102871c91SPatrick Williams#!/usr/bin/env python3 202871c91SPatrick Williams# 302871c91SPatrick Williams# Build the required docker image to run package unit tests 402871c91SPatrick Williams# 502871c91SPatrick Williams# Script Variables: 602871c91SPatrick Williams# DOCKER_IMG_NAME: <optional, the name of the docker image to generate> 702871c91SPatrick Williams# default is openbmc/ubuntu-unit-test 802871c91SPatrick Williams# DISTRO: <optional, the distro to build a docker image against> 950837436SPatrick Williams# default is ubuntu:focal 1050837436SPatrick Williams# FORCE_DOCKER_BUILD: <optional, a non-zero value with force all Docker 1150837436SPatrick Williams# images to be rebuilt rather than reusing caches.> 1250837436SPatrick Williams# BUILD_URL: <optional, used to detect running under CI context 1350837436SPatrick Williams# (ex. Jenkins)> 1402871c91SPatrick Williams# BRANCH: <optional, branch to build from each of the openbmc/ 1502871c91SPatrick Williams# repositories> 1602871c91SPatrick Williams# default is master, which will be used if input branch not 1702871c91SPatrick Williams# provided or not found 1802871c91SPatrick Williams# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the 1902871c91SPatrick Williams# default ones in /etc/apt/sources.list> 2002871c91SPatrick Williams# default is empty, and no mirror is used. 2102871c91SPatrick Williams# http_proxy The HTTP address of the proxy server to connect to. 2202871c91SPatrick Williams# Default: "", proxy is not setup if this is not set 2302871c91SPatrick Williams 2402871c91SPatrick Williamsimport os 2502871c91SPatrick Williamsimport sys 26b16f3e20SPatrick Williamsimport threading 27a18d9c57SPatrick Williamsfrom datetime import date 28a18d9c57SPatrick Williamsfrom hashlib import sha256 2902871c91SPatrick Williamsfrom sh import docker, git, nproc, uname 3002871c91SPatrick Williams 3102871c91SPatrick Williams# Read a bunch of environment variables. 3202871c91SPatrick Williamsdocker_image_name = os.environ.get("DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test") 3350837436SPatrick Williamsforce_build = os.environ.get("FORCE_DOCKER_BUILD") 3450837436SPatrick Williamsis_automated_ci_build = os.environ.get("BUILD_URL", False) 3502871c91SPatrick Williamsdistro = os.environ.get("DISTRO", "ubuntu:focal") 3602871c91SPatrick Williamsbranch = os.environ.get("BRANCH", "master") 3702871c91SPatrick Williamsubuntu_mirror = os.environ.get("UBUNTU_MIRROR") 3802871c91SPatrick Williamshttp_proxy = os.environ.get("http_proxy") 39aae36d18SPatrick Williamsprefix = "/usr/local" 4002871c91SPatrick Williams 4102871c91SPatrick Williams# Set up some common variables. 4202871c91SPatrick Williamsproc_count = nproc().strip() 4302871c91SPatrick Williamsusername = os.environ.get("USER") 4402871c91SPatrick Williamshomedir = os.environ.get("HOME") 4502871c91SPatrick Williamsgid = os.getgid() 4602871c91SPatrick Williamsuid = os.getuid() 4702871c91SPatrick Williams 4802871c91SPatrick Williams# Determine the architecture for Docker. 4902871c91SPatrick Williamsarch = uname("-m").strip() 5002871c91SPatrick Williamsif arch == "ppc64le": 5102871c91SPatrick Williams docker_base = "ppc64le/" 5202871c91SPatrick Williamselif arch == "x86_64": 5302871c91SPatrick Williams docker_base = "" 5402871c91SPatrick Williamselse: 55b16f3e20SPatrick Williams print( 56b16f3e20SPatrick Williams f"Unsupported system architecture({arch}) found for docker image", 57b16f3e20SPatrick Williams file=sys.stderr, 58b16f3e20SPatrick Williams ) 5902871c91SPatrick Williams sys.exit(1) 6002871c91SPatrick Williams 617204324cSPatrick Williams# Packages to include in image. 627204324cSPatrick Williamspackages = { 632abc4a48SPatrick Williams "boost": { 64*0eedeedaSPatrick Williams "rev": "1.75.0", 652abc4a48SPatrick Williams "url": ( 662abc4a48SPatrick Williams lambda pkg, rev: f"https://dl.bintray.com/boostorg/release/{rev}/source/{pkg}_{rev.replace('.', '_')}.tar.bz2" 672abc4a48SPatrick Williams ), 68aae36d18SPatrick Williams "build_type": "custom", 69aae36d18SPatrick Williams "build_steps": [ 70aae36d18SPatrick Williams f"./bootstrap.sh --prefix={prefix} --with-libraries=context,coroutine", 71aae36d18SPatrick Williams "./b2", 72aae36d18SPatrick Williams f"./b2 install --prefix={prefix}", 73aae36d18SPatrick Williams ], 742abc4a48SPatrick Williams }, 75aae36d18SPatrick Williams "USCiLab/cereal": { 76aae36d18SPatrick Williams "rev": "v1.3.0", 77aae36d18SPatrick Williams "build_type": "custom", 78aae36d18SPatrick Williams "build_steps": [f"cp -a include/cereal/ {prefix}/include/"], 79aae36d18SPatrick Williams }, 80aae36d18SPatrick Williams "catchorg/Catch2": { 81aae36d18SPatrick Williams "rev": "v2.12.2", 82aae36d18SPatrick Williams "build_type": "cmake", 83aae36d18SPatrick Williams "config_flags": ["-DBUILD_TESTING=OFF", "-DCATCH_INSTALL_DOCS=OFF"], 84aae36d18SPatrick Williams }, 85aae36d18SPatrick Williams "CLIUtils/CLI11": { 86*0eedeedaSPatrick Williams "rev": "v1.9.1", 87aae36d18SPatrick Williams "build_type": "cmake", 88aae36d18SPatrick Williams "config_flags": [ 89aae36d18SPatrick Williams "-DBUILD_TESTING=OFF", 90aae36d18SPatrick Williams "-DCLI11_BUILD_DOCS=OFF", 91aae36d18SPatrick Williams "-DCLI11_BUILD_EXAMPLES=OFF", 92aae36d18SPatrick Williams ], 93aae36d18SPatrick Williams }, 94aae36d18SPatrick Williams "fmtlib/fmt": { 95*0eedeedaSPatrick Williams "rev": "7.1.3", 96aae36d18SPatrick Williams "build_type": "cmake", 97aae36d18SPatrick Williams "config_flags": [ 98aae36d18SPatrick Williams "-DFMT_DOC=OFF", 99aae36d18SPatrick Williams "-DFMT_TEST=OFF", 100aae36d18SPatrick Williams ], 101aae36d18SPatrick Williams }, 10202871c91SPatrick Williams # Snapshot from 2020-01-03 103aae36d18SPatrick Williams "Naios/function2": { 104aae36d18SPatrick Williams "rev": "3a0746bf5f601dfed05330aefcb6854354fce07d", 105aae36d18SPatrick Williams "build_type": "custom", 106aae36d18SPatrick Williams "build_steps": [ 107aae36d18SPatrick Williams f"mkdir {prefix}/include/function2", 108aae36d18SPatrick Williams f"cp include/function2/function2.hpp {prefix}/include/function2/", 109aae36d18SPatrick Williams ], 110aae36d18SPatrick Williams }, 111aae36d18SPatrick Williams "google/googletest": { 112*0eedeedaSPatrick Williams "rev": "release-1.10.0", 113aae36d18SPatrick Williams "build_type": "cmake", 114aae36d18SPatrick Williams "config_env": ["CXXFLAGS=-std=c++17"], 115aae36d18SPatrick Williams "config_flags": ["-DTHREADS_PREFER_PTHREAD_FLAG=ON"], 116aae36d18SPatrick Williams }, 11702871c91SPatrick Williams # Release 2020-08-06 118aae36d18SPatrick Williams "nlohmann/json": { 119aae36d18SPatrick Williams "rev": "v3.9.1", 120aae36d18SPatrick Williams "build_type": "custom", 121aae36d18SPatrick Williams "build_steps": [ 122aae36d18SPatrick Williams f"mkdir {prefix}/include/nlohmann", 123*0eedeedaSPatrick Williams f"cp single_include/nlohmann/json.hpp {prefix}/include/nlohmann", 124aae36d18SPatrick Williams f"ln -s {prefix}/include/nlohmann/json.hpp {prefix}/include/json.hpp", 125aae36d18SPatrick Williams ], 126aae36d18SPatrick Williams }, 12702871c91SPatrick Williams # Snapshot from 2019-05-24 128aae36d18SPatrick Williams "linux-test-project/lcov": { 129*0eedeedaSPatrick Williams "rev": "v1.15", 130aae36d18SPatrick Williams "build_type": "make", 131aae36d18SPatrick Williams }, 132*0eedeedaSPatrick Williams # dev-5.8 2021-01-11 133aae36d18SPatrick Williams "openbmc/linux": { 134*0eedeedaSPatrick Williams "rev": "3cc95ae40716e56f81b69615781f54c78079042d", 135aae36d18SPatrick Williams "build_type": "custom", 136aae36d18SPatrick Williams "build_steps": [ 137aae36d18SPatrick Williams f"make -j{proc_count} defconfig", 138aae36d18SPatrick Williams f"make INSTALL_HDR_PATH={prefix} headers_install", 139aae36d18SPatrick Williams ], 140aae36d18SPatrick Williams }, 141*0eedeedaSPatrick Williams # Snapshot from 2020-06-13 142aae36d18SPatrick Williams "LibVNC/libvncserver": { 143*0eedeedaSPatrick Williams "rev": "LibVNCServer-0.9.13", 144aae36d18SPatrick Williams "build_type": "cmake", 145aae36d18SPatrick Williams }, 146aae36d18SPatrick Williams "martinmoene/span-lite": { 147*0eedeedaSPatrick Williams "rev": "v0.8.1", 148aae36d18SPatrick Williams "build_type": "cmake", 149aae36d18SPatrick Williams "config_flags": [ 150aae36d18SPatrick Williams "-DSPAN_LITE_OPT_BUILD_TESTS=OFF", 151aae36d18SPatrick Williams ], 152aae36d18SPatrick Williams }, 153*0eedeedaSPatrick Williams # version from meta-openembedded/meta-oe/recipes-support/libtinyxml2/libtinyxml2_8.0.0.bb 154aae36d18SPatrick Williams "leethomason/tinyxml2": { 155*0eedeedaSPatrick Williams "rev": "8.0.0", 156aae36d18SPatrick Williams "build_type": "cmake", 157aae36d18SPatrick Williams }, 15802871c91SPatrick Williams # version from /meta-openembedded/meta-oe/recipes-devtools/boost-url/boost-url_git.bb 159aae36d18SPatrick Williams "CPPAlliance/url": { 160aae36d18SPatrick Williams "rev": "a56ae0df6d3078319755fbaa67822b4fa7fd352b", 161aae36d18SPatrick Williams "build_type": "cmake", 162aae36d18SPatrick Williams "config_flags": [ 163aae36d18SPatrick Williams "-DBOOST_URL_BUILD_EXAMPLES=OFF", 164aae36d18SPatrick Williams "-DBOOST_URL_BUILD_TESTS=OFF", 165aae36d18SPatrick Williams "-DBOOST_URL_STANDALONE=ON", 166aae36d18SPatrick Williams ], 167aae36d18SPatrick Williams }, 168*0eedeedaSPatrick Williams # version from ./meta-openembedded/meta-oe/dynamic-layers/networking-layer/recipes-devtools/valijson/valijson_0.3.bb 169*0eedeedaSPatrick Williams # Snapshot from 2020-12-02 - fix for curlpp dependency 170aae36d18SPatrick Williams "tristanpenman/valijson": { 171*0eedeedaSPatrick Williams "rev": "8cc83c8be9c1c927f5da952b2333b30e5f0353be", 172aae36d18SPatrick Williams "build_type": "cmake", 173aae36d18SPatrick Williams "config_flags": [ 174*0eedeedaSPatrick Williams "-Dvalijson_BUILD_TESTS=0", 175*0eedeedaSPatrick Williams "-Dvalijson_INSTALL_HEADERS=1", 176aae36d18SPatrick Williams ], 177aae36d18SPatrick Williams }, 17802871c91SPatrick Williams # version from meta-openembedded/meta-oe/recipes-devtools/nlohmann-fifo/nlohmann-fifo_git.bb 179aae36d18SPatrick Williams "nlohmann/fifo_map": { 180aae36d18SPatrick Williams "rev": "0dfbf5dacbb15a32c43f912a7e66a54aae39d0f9", 181aae36d18SPatrick Williams "build_type": "custom", 182aae36d18SPatrick Williams "build_steps": [f"cp src/fifo_map.hpp {prefix}/include/"], 183aae36d18SPatrick Williams }, 184aae36d18SPatrick Williams "open-power/pdbg": {"build_type": "autoconf"}, 185aae36d18SPatrick Williams "openbmc/gpioplus": { 186aae36d18SPatrick Williams "depends": ["openbmc/stdplus"], 187aae36d18SPatrick Williams "build_type": "meson", 188aae36d18SPatrick Williams "config_flags": [ 189aae36d18SPatrick Williams "-Dexamples=false", 190aae36d18SPatrick Williams "-Dtests=disabled", 191aae36d18SPatrick Williams ], 192aae36d18SPatrick Williams }, 193aae36d18SPatrick Williams "openbmc/phosphor-dbus-interfaces": { 194aae36d18SPatrick Williams "depends": ["openbmc/sdbusplus"], 195aae36d18SPatrick Williams "build_type": "meson", 196aae36d18SPatrick Williams "config_flags": [ 197aae36d18SPatrick Williams "-Ddata_com_ibm=true", 198aae36d18SPatrick Williams "-Ddata_org_open_power=true", 199aae36d18SPatrick Williams ], 200aae36d18SPatrick Williams }, 20183394610SPatrick Williams "openbmc/phosphor-logging": { 20283394610SPatrick Williams "depends": [ 20383394610SPatrick Williams "USCiLab/cereal", 20483394610SPatrick Williams "nlohmann/fifo_map", 20583394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 20683394610SPatrick Williams "openbmc/sdbusplus", 20783394610SPatrick Williams "openbmc/sdeventplus", 208aae36d18SPatrick Williams ], 209aae36d18SPatrick Williams "build_type": "autoconf", 210aae36d18SPatrick Williams "config_flags": [ 211aae36d18SPatrick Williams "--enable-metadata-processing", 212aae36d18SPatrick Williams f"YAML_DIR={prefix}/share/phosphor-dbus-yaml/yaml", 213aae36d18SPatrick Williams ], 21483394610SPatrick Williams }, 21583394610SPatrick Williams "openbmc/phosphor-objmgr": { 21683394610SPatrick Williams "depends": [ 21783394610SPatrick Williams "boost", 21883394610SPatrick Williams "leethomason/tinyxml2", 21983394610SPatrick Williams "openbmc/phosphor-logging", 22083394610SPatrick Williams "openbmc/sdbusplus", 221aae36d18SPatrick Williams ], 222aae36d18SPatrick Williams "build_type": "autoconf", 22383394610SPatrick Williams }, 22483394610SPatrick Williams "openbmc/pldm": { 22583394610SPatrick Williams "depends": [ 22683394610SPatrick Williams "CLIUtils/CLI11", 22783394610SPatrick Williams "boost", 22883394610SPatrick Williams "nlohmann/json", 22983394610SPatrick Williams "openbmc/phosphor-dbus-interfaces", 23083394610SPatrick Williams "openbmc/phosphor-logging", 23183394610SPatrick Williams "openbmc/sdbusplus", 23283394610SPatrick Williams "openbmc/sdeventplus", 233aae36d18SPatrick Williams ], 234aae36d18SPatrick Williams "build_type": "meson", 235aae36d18SPatrick Williams "config_flags": [ 236aae36d18SPatrick Williams "-Dlibpldm-only=enabled", 237aae36d18SPatrick Williams "-Doem-ibm=enabled", 238aae36d18SPatrick Williams "-Dtests=disabled", 239aae36d18SPatrick Williams ], 24083394610SPatrick Williams }, 241aae36d18SPatrick Williams "openbmc/sdbusplus": { 242aae36d18SPatrick Williams "build_type": "meson", 243aae36d18SPatrick Williams "custom_post_dl": [ 244aae36d18SPatrick Williams "cd tools", 245aae36d18SPatrick Williams f"./setup.py install --root=/ --prefix={prefix}", 246aae36d18SPatrick Williams "cd ..", 247aae36d18SPatrick Williams ], 248aae36d18SPatrick Williams "config_flags": [ 249aae36d18SPatrick Williams "-Dexamples=disabled", 250aae36d18SPatrick Williams "-Dtests=disabled", 251aae36d18SPatrick Williams ], 252aae36d18SPatrick Williams }, 253aae36d18SPatrick Williams "openbmc/sdeventplus": { 254aae36d18SPatrick Williams "depends": ["Naios/function2", "openbmc/stdplus"], 255aae36d18SPatrick Williams "build_type": "meson", 256aae36d18SPatrick Williams "config_flags": [ 257aae36d18SPatrick Williams "-Dexamples=false", 258aae36d18SPatrick Williams "-Dtests=disabled", 259aae36d18SPatrick Williams ], 260aae36d18SPatrick Williams }, 261aae36d18SPatrick Williams "openbmc/stdplus": { 262aae36d18SPatrick Williams "depends": ["fmtlib/fmt", "martinmoene/span-lite"], 263aae36d18SPatrick Williams "build_type": "meson", 264aae36d18SPatrick Williams "config_flags": [ 265aae36d18SPatrick Williams "-Dexamples=false", 266aae36d18SPatrick Williams "-Dtests=disabled", 267aae36d18SPatrick Williams ], 268aae36d18SPatrick Williams }, 26902871c91SPatrick Williams} 27002871c91SPatrick Williams 2712abc4a48SPatrick Williams 2727204324cSPatrick Williamsdef pkg_rev(pkg): 2737204324cSPatrick Williams return packages[pkg]["rev"] 2747204324cSPatrick Williams 2752abc4a48SPatrick Williams 276e1398742SPatrick Williamsdef pkg_stagename(pkg): 277a18d9c57SPatrick Williams return pkg.replace("/", "-").lower() 2787204324cSPatrick Williams 2792abc4a48SPatrick Williams 2802abc4a48SPatrick Williamsdef pkg_url(pkg): 2812abc4a48SPatrick Williams if "url" in packages[pkg]: 2822abc4a48SPatrick Williams return packages[pkg]["url"](pkg, pkg_rev(pkg)) 2832abc4a48SPatrick Williams return f"https://github.com/{pkg}/archive/{pkg_rev(pkg)}.tar.gz" 2842abc4a48SPatrick Williams 2852abc4a48SPatrick Williams 286aae36d18SPatrick Williamsdef pkg_download(pkg): 287aae36d18SPatrick Williams url = pkg_url(pkg) 288aae36d18SPatrick Williams if ".tar." not in url: 289aae36d18SPatrick Williams raise NotImplementedError(f"Unhandled download type for {pkg}: {url}") 290aae36d18SPatrick Williams cmd = f"curl -L {url} | tar -x" 291aae36d18SPatrick Williams if url.endswith(".bz2"): 292aae36d18SPatrick Williams cmd += "j" 293aae36d18SPatrick Williams if url.endswith(".gz"): 294aae36d18SPatrick Williams cmd += "z" 295aae36d18SPatrick Williams return cmd 296aae36d18SPatrick Williams 297aae36d18SPatrick Williams 29883394610SPatrick Williamsdef pkg_copycmds(pkg=None): 29983394610SPatrick Williams pkgs = [] 30083394610SPatrick Williams if pkg: 30183394610SPatrick Williams if "depends" not in packages[pkg]: 30283394610SPatrick Williams return "" 30383394610SPatrick Williams pkgs = sorted(packages[pkg]["depends"]) 30483394610SPatrick Williams else: 30583394610SPatrick Williams pkgs = sorted(packages.keys()) 30683394610SPatrick Williams 30783394610SPatrick Williams copy_cmds = "" 30883394610SPatrick Williams for p in pkgs: 309a18d9c57SPatrick Williams copy_cmds += f"COPY --from={packages[p]['__tag']} {prefix} {prefix}\n" 31083394610SPatrick Williams # Workaround for upstream docker bug and multiple COPY cmds 31183394610SPatrick Williams # https://github.com/moby/moby/issues/37965 31283394610SPatrick Williams copy_cmds += "RUN true\n" 31383394610SPatrick Williams return copy_cmds 31483394610SPatrick Williams 31583394610SPatrick Williams 316aae36d18SPatrick Williamsdef pkg_cd_srcdir(pkg): 317aae36d18SPatrick Williams return f"cd {pkg.split('/')[-1]}* && " 318aae36d18SPatrick Williams 319aae36d18SPatrick Williams 320aae36d18SPatrick Williamsdef pkg_build(pkg): 321aae36d18SPatrick Williams result = f"RUN {pkg_download(pkg)} && " 322aae36d18SPatrick Williams result += pkg_cd_srcdir(pkg) 323aae36d18SPatrick Williams 324aae36d18SPatrick Williams if "custom_post_dl" in packages[pkg]: 325aae36d18SPatrick Williams result += " && ".join(packages[pkg]["custom_post_dl"]) + " && " 326aae36d18SPatrick Williams 327aae36d18SPatrick Williams build_type = packages[pkg]["build_type"] 328aae36d18SPatrick Williams if build_type == "autoconf": 329aae36d18SPatrick Williams result += pkg_build_autoconf(pkg) 330aae36d18SPatrick Williams elif build_type == "cmake": 331aae36d18SPatrick Williams result += pkg_build_cmake(pkg) 332aae36d18SPatrick Williams elif build_type == "custom": 333aae36d18SPatrick Williams result += pkg_build_custom(pkg) 334aae36d18SPatrick Williams elif build_type == "make": 335aae36d18SPatrick Williams result += pkg_build_make(pkg) 336aae36d18SPatrick Williams elif build_type == "meson": 337aae36d18SPatrick Williams result += pkg_build_meson(pkg) 338aae36d18SPatrick Williams else: 339aae36d18SPatrick Williams raise NotImplementedError( 340aae36d18SPatrick Williams f"Unhandled build type for {pkg}: {packages[pkg]['build_type']}" 341aae36d18SPatrick Williams ) 342aae36d18SPatrick Williams 343aae36d18SPatrick Williams return result 344aae36d18SPatrick Williams 345aae36d18SPatrick Williams 346aae36d18SPatrick Williamsdef pkg_build_autoconf(pkg): 347aae36d18SPatrick Williams options = " ".join(packages[pkg].get("config_flags", [])) 348aae36d18SPatrick Williams env = " ".join(packages[pkg].get("config_env", [])) 349aae36d18SPatrick Williams result = "./bootstrap.sh && " 350aae36d18SPatrick Williams result += f"{env} ./configure {configure_flags} {options} && " 351aae36d18SPatrick Williams result += f"make -j{proc_count} && " 352aae36d18SPatrick Williams result += "make install " 353aae36d18SPatrick Williams return result 354aae36d18SPatrick Williams 355aae36d18SPatrick Williams 356aae36d18SPatrick Williamsdef pkg_build_cmake(pkg): 357aae36d18SPatrick Williams options = " ".join(packages[pkg].get("config_flags", [])) 358aae36d18SPatrick Williams env = " ".join(packages[pkg].get("config_env", [])) 359aae36d18SPatrick Williams result = "mkdir builddir && cd builddir && " 360aae36d18SPatrick Williams result += f"{env} cmake {cmake_flags} {options} .. && " 3610f2086b3SPatrick Williams result += "cmake --build . --target all && " 3620f2086b3SPatrick Williams result += "cmake --build . --target install && " 363aae36d18SPatrick Williams result += "cd .. " 364aae36d18SPatrick Williams return result 365aae36d18SPatrick Williams 366aae36d18SPatrick Williams 367aae36d18SPatrick Williamsdef pkg_build_custom(pkg): 368aae36d18SPatrick Williams return " && ".join(packages[pkg].get("build_steps", [])) 369aae36d18SPatrick Williams 370aae36d18SPatrick Williams 371aae36d18SPatrick Williamsdef pkg_build_make(pkg): 372aae36d18SPatrick Williams result = f"make -j{proc_count} && " 373aae36d18SPatrick Williams result += "make install " 374aae36d18SPatrick Williams return result 375aae36d18SPatrick Williams 376aae36d18SPatrick Williams 377aae36d18SPatrick Williamsdef pkg_build_meson(pkg): 378aae36d18SPatrick Williams options = " ".join(packages[pkg].get("config_flags", [])) 379aae36d18SPatrick Williams env = " ".join(packages[pkg].get("config_env", [])) 380aae36d18SPatrick Williams result = f"{env} meson builddir {meson_flags} {options} && " 381aae36d18SPatrick Williams result += "ninja -C builddir && ninja -C builddir install " 382aae36d18SPatrick Williams return result 383aae36d18SPatrick Williams 384aae36d18SPatrick Williams 385b16f3e20SPatrick Williamspkg_lock = threading.Lock() 3868751b80eSPatrick Williams 387b16f3e20SPatrick Williams 388b16f3e20SPatrick Williamsdef pkg_generate(pkg): 389b16f3e20SPatrick Williams class pkg_thread(threading.Thread): 390b16f3e20SPatrick Williams def run(self): 391b16f3e20SPatrick Williams pkg_lock.acquire() 392b16f3e20SPatrick Williams deps = [ 393b16f3e20SPatrick Williams packages[deppkg]["__thread"] 394b16f3e20SPatrick Williams for deppkg in sorted(packages[pkg].get("depends", [])) 395b16f3e20SPatrick Williams ] 396b16f3e20SPatrick Williams pkg_lock.release() 397b16f3e20SPatrick Williams for deppkg in deps: 398b16f3e20SPatrick Williams deppkg.join() 3998751b80eSPatrick Williams 400a18d9c57SPatrick Williams dockerfile = f""" 401a18d9c57SPatrick WilliamsFROM {docker_base_img_name} 402a18d9c57SPatrick Williams{pkg_copycmds(pkg)} 403a18d9c57SPatrick Williams{pkg_build(pkg)} 404a18d9c57SPatrick Williams""" 405a18d9c57SPatrick Williams 406b16f3e20SPatrick Williams pkg_lock.acquire() 407a18d9c57SPatrick Williams tag = docker_img_tagname(pkg_stagename(pkg), dockerfile) 408a18d9c57SPatrick Williams packages[pkg]["__tag"] = tag 409b16f3e20SPatrick Williams pkg_lock.release() 4108751b80eSPatrick Williams 411b16f3e20SPatrick Williams try: 412b16f3e20SPatrick Williams self.exception = None 413b16f3e20SPatrick Williams docker_img_build(pkg, tag, dockerfile) 414b16f3e20SPatrick Williams except Exception as e: 415b16f3e20SPatrick Williams self.package = pkg 416b16f3e20SPatrick Williams self.exception = e 417b16f3e20SPatrick Williams 418b16f3e20SPatrick Williams packages[pkg]["__thread"] = pkg_thread() 4198751b80eSPatrick Williams 4208751b80eSPatrick Williams 4218751b80eSPatrick Williamsdef pkg_generate_packages(): 422b16f3e20SPatrick Williams for pkg in packages.keys(): 423a18d9c57SPatrick Williams pkg_generate(pkg) 424a18d9c57SPatrick Williams 425b16f3e20SPatrick Williams pkg_lock.acquire() 426b16f3e20SPatrick Williams pkg_threads = [packages[p]["__thread"] for p in packages.keys()] 427b16f3e20SPatrick Williams for t in pkg_threads: 428b16f3e20SPatrick Williams t.start() 429b16f3e20SPatrick Williams pkg_lock.release() 430b16f3e20SPatrick Williams 431b16f3e20SPatrick Williams for t in pkg_threads: 432b16f3e20SPatrick Williams t.join() 433b16f3e20SPatrick Williams if t.exception: 434b16f3e20SPatrick Williams print(f"Package {t.package} failed!", file=sys.stderr) 435b16f3e20SPatrick Williams raise t.exception 436b16f3e20SPatrick Williams 43750837436SPatrick Williamsdef timestamp(): 43850837436SPatrick Williams today = date.today().isocalendar() 43950837436SPatrick Williams return f"{today[0]}-W{today[1]:02}" 440a18d9c57SPatrick Williams 441a18d9c57SPatrick Williamsdef docker_img_tagname(pkgname, dockerfile): 442a18d9c57SPatrick Williams result = docker_image_name 443a18d9c57SPatrick Williams if pkgname: 444a18d9c57SPatrick Williams result += "-" + pkgname 44550837436SPatrick Williams result += ":" + timestamp() 446a18d9c57SPatrick Williams result += "-" + sha256(dockerfile.encode()).hexdigest()[0:16] 4478751b80eSPatrick Williams return result 4488751b80eSPatrick Williams 4498751b80eSPatrick Williams 450b16f3e20SPatrick Williamsdef docker_img_build(pkg, tag, dockerfile): 45100536fbeSPatrick Williams if not force_build: 45250837436SPatrick Williams if docker.image.ls(tag, "--format", '"{{.Repository}}:{{.Tag}}"'): 45350837436SPatrick Williams print(f"Image {tag} already exists. Skipping.", file=sys.stderr) 45450837436SPatrick Williams return 45550837436SPatrick Williams 456b16f3e20SPatrick Williams docker.build( 457a18d9c57SPatrick Williams proxy_args, 458a18d9c57SPatrick Williams "--network=host", 459a18d9c57SPatrick Williams "--force-rm", 46050837436SPatrick Williams "--no-cache=true" if force_build else "--no-cache=false", 461a18d9c57SPatrick Williams "-t", 462a18d9c57SPatrick Williams tag, 463a18d9c57SPatrick Williams "-", 464a18d9c57SPatrick Williams _in=dockerfile, 465b16f3e20SPatrick Williams _out=( 466b16f3e20SPatrick Williams lambda line: print(pkg + ":", line, end="", file=sys.stderr, flush=True) 467b16f3e20SPatrick Williams ), 468b16f3e20SPatrick Williams ) 469a18d9c57SPatrick Williams 470a18d9c57SPatrick Williams 4717204324cSPatrick Williams# Look up the HEAD for missing a static rev. 47202871c91SPatrick Williamspkg_lookups = {} 4737204324cSPatrick Williamsfor pkg in packages.keys(): 4747204324cSPatrick Williams if "rev" in packages[pkg]: 4757204324cSPatrick Williams continue 47602871c91SPatrick Williams pkg_lookups[pkg] = git( 47702871c91SPatrick Williams "ls-remote", "--heads", f"https://github.com/{pkg}", _bg=True 47802871c91SPatrick Williams ) 47902871c91SPatrick Williamsfor pkg, result in pkg_lookups.items(): 48002871c91SPatrick Williams for line in result.stdout.decode().split("\n"): 48102871c91SPatrick Williams if f"refs/heads/{branch}" in line: 4827204324cSPatrick Williams packages[pkg]["rev"] = line.strip().split()[0] 4837204324cSPatrick Williams elif "refs/heads/master" in line and p not in packages: 4847204324cSPatrick Williams packages[pkg]["rev"] = line.strip().split()[0] 48502871c91SPatrick Williams 48602871c91SPatrick Williams# Create the contents of the '/tmp/depcache'. 48702871c91SPatrick Williams# This needs to be sorted for consistency. 48802871c91SPatrick Williamsdepcache = "" 4897204324cSPatrick Williamsfor pkg in sorted(packages.keys()): 4907204324cSPatrick Williams depcache += "%s:%s," % (pkg, pkg_rev(pkg)) 49102871c91SPatrick Williams 49202871c91SPatrick Williams# Define common flags used for builds 49302871c91SPatrick Williamsconfigure_flags = " ".join( 49402871c91SPatrick Williams [ 49502871c91SPatrick Williams f"--prefix={prefix}", 49602871c91SPatrick Williams ] 49702871c91SPatrick Williams) 49802871c91SPatrick Williamscmake_flags = " ".join( 49902871c91SPatrick Williams [ 50002871c91SPatrick Williams "-DBUILD_SHARED_LIBS=ON", 5010f2086b3SPatrick Williams "-DCMAKE_BUILD_TYPE=RelWithDebInfo", 50202871c91SPatrick Williams f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}", 5030f2086b3SPatrick Williams "-GNinja", 5040f2086b3SPatrick Williams "-DCMAKE_MAKE_PROGRAM=ninja", 50502871c91SPatrick Williams ] 50602871c91SPatrick Williams) 50702871c91SPatrick Williamsmeson_flags = " ".join( 50802871c91SPatrick Williams [ 50902871c91SPatrick Williams "--wrap-mode=nodownload", 51002871c91SPatrick Williams f"-Dprefix={prefix}", 51102871c91SPatrick Williams ] 51202871c91SPatrick Williams) 51302871c91SPatrick Williams 51402871c91SPatrick Williams# Special flags if setting up a deb mirror. 51502871c91SPatrick Williamsmirror = "" 51602871c91SPatrick Williamsif "ubuntu" in distro and ubuntu_mirror: 51702871c91SPatrick Williams mirror = f""" 51802871c91SPatrick WilliamsRUN echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME) main restricted universe multiverse" > /etc/apt/sources.list && \\ 51902871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-updates main restricted universe multiverse" >> /etc/apt/sources.list && \\ 52002871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-security main restricted universe multiverse" >> /etc/apt/sources.list && \\ 52102871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-proposed main restricted universe multiverse" >> /etc/apt/sources.list && \\ 52202871c91SPatrick Williams echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-backports main restricted universe multiverse" >> /etc/apt/sources.list 52302871c91SPatrick Williams""" 52402871c91SPatrick Williams 52502871c91SPatrick Williams# Special flags for proxying. 52602871c91SPatrick Williamsproxy_cmd = "" 52702871c91SPatrick Williamsproxy_args = [] 52802871c91SPatrick Williamsif http_proxy: 52902871c91SPatrick Williams proxy_cmd = f""" 53002871c91SPatrick WilliamsRUN echo "[http]" >> {homedir}/.gitconfig && \ 53102871c91SPatrick Williams echo "proxy = {http_proxy}" >> {homedir}/.gitconfig 53202871c91SPatrick Williams""" 53302871c91SPatrick Williams proxy_args.extend( 53402871c91SPatrick Williams [ 53502871c91SPatrick Williams "--build-arg", 53602871c91SPatrick Williams f"http_proxy={http_proxy}", 53702871c91SPatrick Williams "--build-arg", 53802871c91SPatrick Williams "https_proxy={https_proxy}", 53902871c91SPatrick Williams ] 54002871c91SPatrick Williams ) 54102871c91SPatrick Williams 54202871c91SPatrick Williams# Create docker image that can run package unit tests 543a18d9c57SPatrick Williamsdockerfile_base = f""" 544a18d9c57SPatrick WilliamsFROM {docker_base}{distro} 54502871c91SPatrick Williams 54602871c91SPatrick Williams{mirror} 54702871c91SPatrick Williams 54802871c91SPatrick WilliamsENV DEBIAN_FRONTEND noninteractive 54902871c91SPatrick Williams 55002871c91SPatrick WilliamsENV PYTHONPATH "/usr/local/lib/python3.8/site-packages/" 55102871c91SPatrick Williams 55202871c91SPatrick Williams# We need the keys to be imported for dbgsym repos 55302871c91SPatrick Williams# New releases have a package, older ones fall back to manual fetching 55402871c91SPatrick Williams# https://wiki.ubuntu.com/Debug%20Symbol%20Packages 55550837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && \ 55650837436SPatrick Williams ( apt-get install ubuntu-dbgsym-keyring || \ 55750837436SPatrick Williams ( apt-get install -yy dirmngr && \ 55850837436SPatrick Williams apt-key adv --keyserver keyserver.ubuntu.com \ 55950837436SPatrick Williams --recv-keys F2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622 ) ) 56002871c91SPatrick Williams 56102871c91SPatrick Williams# Parse the current repo list into a debug repo list 56202871c91SPatrick WilliamsRUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list 56302871c91SPatrick Williams 56402871c91SPatrick Williams# Remove non-existent debug repos 56502871c91SPatrick WilliamsRUN sed -i '/-\(backports\|security\) /d' /etc/apt/sources.list.d/debug.list 56602871c91SPatrick Williams 56702871c91SPatrick WilliamsRUN cat /etc/apt/sources.list.d/debug.list 56802871c91SPatrick Williams 56902871c91SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \ 57002871c91SPatrick Williams gcc-10 \ 57102871c91SPatrick Williams g++-10 \ 57202871c91SPatrick Williams libc6-dbg \ 57302871c91SPatrick Williams libc6-dev \ 57402871c91SPatrick Williams libtool \ 57502871c91SPatrick Williams bison \ 57602871c91SPatrick Williams libdbus-1-dev \ 57702871c91SPatrick Williams flex \ 57802871c91SPatrick Williams cmake \ 57902871c91SPatrick Williams python3 \ 58002871c91SPatrick Williams python3-dev\ 58102871c91SPatrick Williams python3-yaml \ 58202871c91SPatrick Williams python3-mako \ 58302871c91SPatrick Williams python3-pip \ 58402871c91SPatrick Williams python3-setuptools \ 58502871c91SPatrick Williams python3-git \ 58602871c91SPatrick Williams python3-socks \ 58702871c91SPatrick Williams pkg-config \ 58802871c91SPatrick Williams autoconf \ 58902871c91SPatrick Williams autoconf-archive \ 59002871c91SPatrick Williams libsystemd-dev \ 59102871c91SPatrick Williams systemd \ 59202871c91SPatrick Williams libssl-dev \ 59302871c91SPatrick Williams libevdev-dev \ 59402871c91SPatrick Williams libevdev2-dbgsym \ 59502871c91SPatrick Williams libjpeg-dev \ 59602871c91SPatrick Williams libpng-dev \ 59702871c91SPatrick Williams ninja-build \ 59802871c91SPatrick Williams sudo \ 59902871c91SPatrick Williams curl \ 60002871c91SPatrick Williams git \ 60102871c91SPatrick Williams dbus \ 60202871c91SPatrick Williams iputils-ping \ 60302871c91SPatrick Williams clang-10 \ 60402871c91SPatrick Williams clang-format-10 \ 60502871c91SPatrick Williams clang-tidy-10 \ 60602871c91SPatrick Williams clang-tools-10 \ 60702871c91SPatrick Williams shellcheck \ 60802871c91SPatrick Williams npm \ 60902871c91SPatrick Williams iproute2 \ 61002871c91SPatrick Williams libnl-3-dev \ 61102871c91SPatrick Williams libnl-genl-3-dev \ 61202871c91SPatrick Williams libconfig++-dev \ 61302871c91SPatrick Williams libsnmp-dev \ 61402871c91SPatrick Williams valgrind \ 61502871c91SPatrick Williams valgrind-dbg \ 61602871c91SPatrick Williams libpam0g-dev \ 61702871c91SPatrick Williams xxd \ 61802871c91SPatrick Williams libi2c-dev \ 61902871c91SPatrick Williams wget \ 62002871c91SPatrick Williams libldap2-dev \ 62102871c91SPatrick Williams libprotobuf-dev \ 62202871c91SPatrick Williams libperlio-gzip-perl \ 62302871c91SPatrick Williams libjson-perl \ 62402871c91SPatrick Williams protobuf-compiler \ 62502871c91SPatrick Williams libgpiod-dev \ 62602871c91SPatrick Williams device-tree-compiler \ 62702871c91SPatrick Williams cppcheck \ 62802871c91SPatrick Williams libpciaccess-dev \ 62902871c91SPatrick Williams libmimetic-dev \ 63002871c91SPatrick Williams libxml2-utils \ 631*0eedeedaSPatrick Williams libxml-simple-perl \ 632*0eedeedaSPatrick Williams rsync 63302871c91SPatrick Williams 63402871c91SPatrick WilliamsRUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 1000 \ 63502871c91SPatrick Williams --slave /usr/bin/g++ g++ /usr/bin/g++-10 \ 63602871c91SPatrick Williams --slave /usr/bin/gcov gcov /usr/bin/gcov-10 \ 63702871c91SPatrick Williams --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-10 \ 63802871c91SPatrick Williams --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-10 63902871c91SPatrick Williams 64002871c91SPatrick WilliamsRUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 1000 \ 64102871c91SPatrick Williams --slave /usr/bin/clang++ clang++ /usr/bin/clang++-10 \ 64202871c91SPatrick Williams --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-10 \ 64302871c91SPatrick Williams --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-10 \ 64402871c91SPatrick Williams --slave /usr/bin/run-clang-tidy.py run-clang-tidy.py /usr/bin/run-clang-tidy-10.py 64502871c91SPatrick Williams 64650837436SPatrick Williams""" 64750837436SPatrick Williams 64850837436SPatrick Williamsif is_automated_ci_build: 64950837436SPatrick Williams dockerfile_base += f""" 65050837436SPatrick Williams# Run an arbitrary command to polute the docker cache regularly force us 65150837436SPatrick Williams# to re-run `apt-get update` daily. 65250837436SPatrick WilliamsRUN echo {timestamp()} 65350837436SPatrick WilliamsRUN apt-get update && apt-get dist-upgrade -yy 65450837436SPatrick Williams 65550837436SPatrick Williams""" 65650837436SPatrick Williams 65750837436SPatrick Williamsdockerfile_base += f""" 65802871c91SPatrick WilliamsRUN pip3 install inflection 65902871c91SPatrick WilliamsRUN pip3 install pycodestyle 66002871c91SPatrick WilliamsRUN pip3 install jsonschema 66102871c91SPatrick WilliamsRUN pip3 install meson==0.54.3 66202871c91SPatrick WilliamsRUN pip3 install protobuf 663a18d9c57SPatrick Williams""" 66402871c91SPatrick Williams 665a18d9c57SPatrick Williams# Build the stage docker images. 666a18d9c57SPatrick Williamsdocker_base_img_name = docker_img_tagname("base", dockerfile_base) 667b16f3e20SPatrick Williamsdocker_img_build("base", docker_base_img_name, dockerfile_base) 668a18d9c57SPatrick Williamspkg_generate_packages() 66902871c91SPatrick Williams 670a18d9c57SPatrick Williamsdockerfile = f""" 67102871c91SPatrick Williams# Build the final output image 672a18d9c57SPatrick WilliamsFROM {docker_base_img_name} 673a18d9c57SPatrick Williams{pkg_copycmds()} 67402871c91SPatrick Williams 67502871c91SPatrick Williams# Some of our infrastructure still relies on the presence of this file 67602871c91SPatrick Williams# even though it is no longer needed to rebuild the docker environment 67702871c91SPatrick Williams# NOTE: The file is sorted to ensure the ordering is stable. 67802871c91SPatrick WilliamsRUN echo '{depcache}' > /tmp/depcache 67902871c91SPatrick Williams 68002871c91SPatrick Williams# Final configuration for the workspace 68102871c91SPatrick WilliamsRUN grep -q {gid} /etc/group || groupadd -g {gid} {username} 68202871c91SPatrick WilliamsRUN mkdir -p "{os.path.dirname(homedir)}" 68302871c91SPatrick WilliamsRUN grep -q {uid} /etc/passwd || useradd -d {homedir} -m -u {uid} -g {gid} {username} 68402871c91SPatrick WilliamsRUN sed -i '1iDefaults umask=000' /etc/sudoers 68502871c91SPatrick WilliamsRUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers 68602871c91SPatrick Williams 68702871c91SPatrick Williams{proxy_cmd} 68802871c91SPatrick Williams 68902871c91SPatrick WilliamsRUN /bin/bash 69002871c91SPatrick Williams""" 69102871c91SPatrick Williams 692a18d9c57SPatrick Williams# Do the final docker build 69300536fbeSPatrick Williamsdocker_final_img_name = docker_img_tagname(docker_image_name, dockerfile) 69400536fbeSPatrick Williamsdocker_img_build("final", docker_final_img_name, dockerfile) 69500536fbeSPatrick Williams# Print the tag of the final image. 69600536fbeSPatrick Williamsprint(docker_final_img_name) 697