1#!/usr/bin/env python3 2# 3# Build the required docker image to run package unit tests 4# 5# Script Variables: 6# DOCKER_IMG_NAME: <optional, the name of the docker image to generate> 7# default is openbmc/ubuntu-unit-test 8# DISTRO: <optional, the distro to build a docker image against> 9# default is ubuntu:eoan 10# BRANCH: <optional, branch to build from each of the openbmc/ 11# repositories> 12# default is master, which will be used if input branch not 13# provided or not found 14# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the 15# default ones in /etc/apt/sources.list> 16# default is empty, and no mirror is used. 17# http_proxy The HTTP address of the proxy server to connect to. 18# Default: "", proxy is not setup if this is not set 19 20import os 21import sys 22from sh import docker, git, nproc, uname 23 24# Read a bunch of environment variables. 25docker_image_name = os.environ.get("DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test") 26distro = os.environ.get("DISTRO", "ubuntu:focal") 27branch = os.environ.get("BRANCH", "master") 28ubuntu_mirror = os.environ.get("UBUNTU_MIRROR") 29http_proxy = os.environ.get("http_proxy") 30prefix = "/usr/local" 31 32# Set up some common variables. 33proc_count = nproc().strip() 34username = os.environ.get("USER") 35homedir = os.environ.get("HOME") 36gid = os.getgid() 37uid = os.getuid() 38 39# Determine the architecture for Docker. 40arch = uname("-m").strip() 41if arch == "ppc64le": 42 docker_base = "ppc64le/" 43elif arch == "x86_64": 44 docker_base = "" 45else: 46 print(f"Unsupported system architecture({arch}) found for docker image") 47 sys.exit(1) 48 49# Packages to include in image. 50packages = { 51 "boost": { 52 "rev": "1.74.0", 53 "url": ( 54 lambda pkg, rev: f"https://dl.bintray.com/boostorg/release/{rev}/source/{pkg}_{rev.replace('.', '_')}.tar.bz2" 55 ), 56 "build_type": "custom", 57 "build_steps": [ 58 f"./bootstrap.sh --prefix={prefix} --with-libraries=context,coroutine", 59 "./b2", 60 f"./b2 install --prefix={prefix}", 61 ], 62 }, 63 "USCiLab/cereal": { 64 "rev": "v1.3.0", 65 "build_type": "custom", 66 "build_steps": [f"cp -a include/cereal/ {prefix}/include/"], 67 }, 68 "catchorg/Catch2": { 69 "rev": "v2.12.2", 70 "build_type": "cmake", 71 "config_flags": ["-DBUILD_TESTING=OFF", "-DCATCH_INSTALL_DOCS=OFF"], 72 }, 73 "CLIUtils/CLI11": { 74 "rev": "v1.9.0", 75 "build_type": "cmake", 76 "config_flags": [ 77 "-DBUILD_TESTING=OFF", 78 "-DCLI11_BUILD_DOCS=OFF", 79 "-DCLI11_BUILD_EXAMPLES=OFF", 80 ], 81 }, 82 "fmtlib/fmt": { 83 "rev": "6.2.1", 84 "build_type": "cmake", 85 "config_flags": [ 86 "-DFMT_DOC=OFF", 87 "-DFMT_TEST=OFF", 88 ], 89 }, 90 # Snapshot from 2020-01-03 91 "Naios/function2": { 92 "rev": "3a0746bf5f601dfed05330aefcb6854354fce07d", 93 "build_type": "custom", 94 "build_steps": [ 95 f"mkdir {prefix}/include/function2", 96 f"cp include/function2/function2.hpp {prefix}/include/function2/", 97 ], 98 }, 99 # Snapshot from 2020-02-13 100 "google/googletest": { 101 "rev": "23b2a3b1cf803999fb38175f6e9e038a4495c8a5", 102 "build_type": "cmake", 103 "config_env": ["CXXFLAGS=-std=c++17"], 104 "config_flags": ["-DTHREADS_PREFER_PTHREAD_FLAG=ON"], 105 }, 106 # Release 2020-08-06 107 "nlohmann/json": { 108 "rev": "v3.9.1", 109 "build_type": "custom", 110 "build_steps": [ 111 f"mkdir {prefix}/include/nlohmann", 112 f"cp include/nlohmann/json.hpp {prefix}/include/nlohmann", 113 f"ln -s {prefix}/include/nlohmann/json.hpp {prefix}/include/json.hpp", 114 ], 115 }, 116 # Snapshot from 2019-05-24 117 "linux-test-project/lcov": { 118 "rev": "75fbae1cfc5027f818a0bb865bf6f96fab3202da", 119 "build_type": "make", 120 }, 121 # dev-5.0 2019-05-03 122 "openbmc/linux": { 123 "rev": "8bf6567e77f7aa68975b7c9c6d044bba690bf327", 124 "build_type": "custom", 125 "build_steps": [ 126 f"make -j{proc_count} defconfig", 127 f"make INSTALL_HDR_PATH={prefix} headers_install", 128 ], 129 }, 130 # Snapshot from 2019-09-03 131 "LibVNC/libvncserver": { 132 "rev": "1354f7f1bb6962dab209eddb9d6aac1f03408110", 133 "build_type": "cmake", 134 }, 135 "martinmoene/span-lite": { 136 "rev": "v0.7.0", 137 "build_type": "cmake", 138 "config_flags": [ 139 "-DSPAN_LITE_OPT_BUILD_TESTS=OFF", 140 ], 141 }, 142 # version from meta-openembedded/meta-oe/recipes-support/libtinyxml2/libtinyxml2_5.0.1.bb 143 "leethomason/tinyxml2": { 144 "rev": "37bc3aca429f0164adf68c23444540b4a24b5778", 145 "build_type": "cmake", 146 }, 147 # version from /meta-openembedded/meta-oe/recipes-devtools/boost-url/boost-url_git.bb 148 "CPPAlliance/url": { 149 "rev": "a56ae0df6d3078319755fbaa67822b4fa7fd352b", 150 "build_type": "cmake", 151 "config_flags": [ 152 "-DBOOST_URL_BUILD_EXAMPLES=OFF", 153 "-DBOOST_URL_BUILD_TESTS=OFF", 154 "-DBOOST_URL_STANDALONE=ON", 155 ], 156 }, 157 # version from meta-openembedded/meta-oe/recipes-devtools/valijson/valijson_git.bb 158 "tristanpenman/valijson": { 159 "rev": "c2f22fddf599d04dc33fcd7ed257c698a05345d9", 160 "build_type": "cmake", 161 "config_flags": [ 162 "-DBUILD_TESTS=0", 163 "-DINSTALL_HEADERS=1", 164 ], 165 }, 166 # version from meta-openembedded/meta-oe/recipes-devtools/nlohmann-fifo/nlohmann-fifo_git.bb 167 "nlohmann/fifo_map": { 168 "rev": "0dfbf5dacbb15a32c43f912a7e66a54aae39d0f9", 169 "build_type": "custom", 170 "build_steps": [f"cp src/fifo_map.hpp {prefix}/include/"], 171 }, 172 "open-power/pdbg": {"build_type": "autoconf"}, 173 "openbmc/gpioplus": { 174 "depends": ["openbmc/stdplus"], 175 "build_type": "meson", 176 "config_flags": [ 177 "-Dexamples=false", 178 "-Dtests=disabled", 179 ], 180 }, 181 "openbmc/phosphor-dbus-interfaces": { 182 "depends": ["openbmc/sdbusplus"], 183 "build_type": "meson", 184 "config_flags": [ 185 "-Ddata_com_ibm=true", 186 "-Ddata_org_open_power=true", 187 ], 188 }, 189 "openbmc/phosphor-logging": { 190 "depends": [ 191 "USCiLab/cereal", 192 "nlohmann/fifo_map", 193 "openbmc/phosphor-dbus-interfaces", 194 "openbmc/sdbusplus", 195 "openbmc/sdeventplus", 196 ], 197 "build_type": "autoconf", 198 "config_flags": [ 199 "--enable-metadata-processing", 200 f"YAML_DIR={prefix}/share/phosphor-dbus-yaml/yaml", 201 ], 202 }, 203 "openbmc/phosphor-objmgr": { 204 "depends": [ 205 "boost", 206 "leethomason/tinyxml2", 207 "openbmc/phosphor-logging", 208 "openbmc/sdbusplus", 209 ], 210 "build_type": "autoconf", 211 }, 212 "openbmc/pldm": { 213 "depends": [ 214 "CLIUtils/CLI11", 215 "boost", 216 "nlohmann/json", 217 "openbmc/phosphor-dbus-interfaces", 218 "openbmc/phosphor-logging", 219 "openbmc/sdbusplus", 220 "openbmc/sdeventplus", 221 ], 222 "build_type": "meson", 223 "config_flags": [ 224 "-Dlibpldm-only=enabled", 225 "-Doem-ibm=enabled", 226 "-Dtests=disabled", 227 ], 228 }, 229 "openbmc/sdbusplus": { 230 "build_type": "meson", 231 "custom_post_dl": [ 232 "cd tools", 233 f"./setup.py install --root=/ --prefix={prefix}", 234 "cd ..", 235 ], 236 "config_flags": [ 237 "-Dexamples=disabled", 238 "-Dtests=disabled", 239 ], 240 }, 241 "openbmc/sdeventplus": { 242 "depends": ["Naios/function2", "openbmc/stdplus"], 243 "build_type": "meson", 244 "config_flags": [ 245 "-Dexamples=false", 246 "-Dtests=disabled", 247 ], 248 }, 249 "openbmc/stdplus": { 250 "depends": ["fmtlib/fmt", "martinmoene/span-lite"], 251 "build_type": "meson", 252 "config_flags": [ 253 "-Dexamples=false", 254 "-Dtests=disabled", 255 ], 256 }, 257} 258 259 260def pkg_rev(pkg): 261 return packages[pkg]["rev"] 262 263 264def pkg_stagename(pkg): 265 if not pkg.startswith("openbmc/"): 266 pkg = "openbmc/" + pkg 267 return pkg.replace("/", "-") 268 269 270def pkg_url(pkg): 271 if "url" in packages[pkg]: 272 return packages[pkg]["url"](pkg, pkg_rev(pkg)) 273 return f"https://github.com/{pkg}/archive/{pkg_rev(pkg)}.tar.gz" 274 275 276def pkg_download(pkg): 277 url = pkg_url(pkg) 278 if ".tar." not in url: 279 raise NotImplementedError(f"Unhandled download type for {pkg}: {url}") 280 cmd = f"curl -L {url} | tar -x" 281 if url.endswith(".bz2"): 282 cmd += "j" 283 if url.endswith(".gz"): 284 cmd += "z" 285 return cmd 286 287 288def pkg_copycmds(pkg=None): 289 pkgs = [] 290 if pkg: 291 if "depends" not in packages[pkg]: 292 return "" 293 pkgs = sorted(packages[pkg]["depends"]) 294 else: 295 pkgs = sorted(packages.keys()) 296 297 copy_cmds = "" 298 for p in pkgs: 299 copy_cmds += f"COPY --from={pkg_stagename(p)} {prefix} {prefix}\n" 300 # Workaround for upstream docker bug and multiple COPY cmds 301 # https://github.com/moby/moby/issues/37965 302 copy_cmds += "RUN true\n" 303 return copy_cmds 304 305 306def pkg_cd_srcdir(pkg): 307 return f"cd {pkg.split('/')[-1]}* && " 308 309 310def pkg_build(pkg): 311 result = f"RUN {pkg_download(pkg)} && " 312 result += pkg_cd_srcdir(pkg) 313 314 if "custom_post_dl" in packages[pkg]: 315 result += " && ".join(packages[pkg]["custom_post_dl"]) + " && " 316 317 build_type = packages[pkg]["build_type"] 318 if build_type == "autoconf": 319 result += pkg_build_autoconf(pkg) 320 elif build_type == "cmake": 321 result += pkg_build_cmake(pkg) 322 elif build_type == "custom": 323 result += pkg_build_custom(pkg) 324 elif build_type == "make": 325 result += pkg_build_make(pkg) 326 elif build_type == "meson": 327 result += pkg_build_meson(pkg) 328 else: 329 raise NotImplementedError( 330 f"Unhandled build type for {pkg}: {packages[pkg]['build_type']}" 331 ) 332 333 return result 334 335 336def pkg_build_autoconf(pkg): 337 options = " ".join(packages[pkg].get("config_flags", [])) 338 env = " ".join(packages[pkg].get("config_env", [])) 339 result = "./bootstrap.sh && " 340 result += f"{env} ./configure {configure_flags} {options} && " 341 result += f"make -j{proc_count} && " 342 result += "make install " 343 return result 344 345 346def pkg_build_cmake(pkg): 347 options = " ".join(packages[pkg].get("config_flags", [])) 348 env = " ".join(packages[pkg].get("config_env", [])) 349 result = "mkdir builddir && cd builddir && " 350 result += f"{env} cmake {cmake_flags} {options} .. && " 351 result += "cmake --build . --target all && " 352 result += "cmake --build . --target install && " 353 result += "cd .. " 354 return result 355 356 357def pkg_build_custom(pkg): 358 return " && ".join(packages[pkg].get("build_steps", [])) 359 360 361def pkg_build_make(pkg): 362 result = f"make -j{proc_count} && " 363 result += "make install " 364 return result 365 366 367def pkg_build_meson(pkg): 368 options = " ".join(packages[pkg].get("config_flags", [])) 369 env = " ".join(packages[pkg].get("config_env", [])) 370 result = f"{env} meson builddir {meson_flags} {options} && " 371 result += "ninja -C builddir && ninja -C builddir install " 372 return result 373 374 375def pkg_generate(pkg): 376 if "__pkg_built" in packages[pkg]: 377 return None 378 379 result = "" 380 381 for deppkg in sorted(packages[pkg].get("depends", [])): 382 dep_build = pkg_generate(deppkg) 383 if not dep_build: 384 continue 385 result += dep_build + "\n" 386 387 result += f"FROM openbmc-base as {pkg_stagename(pkg)}\n" 388 result += pkg_copycmds(pkg) 389 result += pkg_build(pkg) 390 391 packages[pkg]["__pkg_built"] = True 392 return result 393 394 395def pkg_generate_packages(): 396 result = "" 397 for pkg in sorted(packages.keys()): 398 build = pkg_generate(pkg) 399 if not build: 400 continue 401 result += build + "\n" 402 return result 403 404 405# Look up the HEAD for missing a static rev. 406pkg_lookups = {} 407for pkg in packages.keys(): 408 if "rev" in packages[pkg]: 409 continue 410 pkg_lookups[pkg] = git( 411 "ls-remote", "--heads", f"https://github.com/{pkg}", _bg=True 412 ) 413for pkg, result in pkg_lookups.items(): 414 for line in result.stdout.decode().split("\n"): 415 if f"refs/heads/{branch}" in line: 416 packages[pkg]["rev"] = line.strip().split()[0] 417 elif "refs/heads/master" in line and p not in packages: 418 packages[pkg]["rev"] = line.strip().split()[0] 419 420# Create the contents of the '/tmp/depcache'. 421# This needs to be sorted for consistency. 422depcache = "" 423for pkg in sorted(packages.keys()): 424 depcache += "%s:%s," % (pkg, pkg_rev(pkg)) 425 426# Define common flags used for builds 427configure_flags = " ".join( 428 [ 429 f"--prefix={prefix}", 430 ] 431) 432cmake_flags = " ".join( 433 [ 434 "-DBUILD_SHARED_LIBS=ON", 435 "-DCMAKE_BUILD_TYPE=RelWithDebInfo", 436 f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}", 437 "-GNinja", 438 "-DCMAKE_MAKE_PROGRAM=ninja", 439 ] 440) 441meson_flags = " ".join( 442 [ 443 "--wrap-mode=nodownload", 444 f"-Dprefix={prefix}", 445 ] 446) 447 448# Build the commands needed to compose our final image 449# We must sort the packages, otherwise we might produce an unstable 450# docker file and rebuild the image unnecessarily 451copy_cmds = pkg_copycmds() 452 453# Special flags if setting up a deb mirror. 454mirror = "" 455if "ubuntu" in distro and ubuntu_mirror: 456 mirror = f""" 457RUN echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME) main restricted universe multiverse" > /etc/apt/sources.list && \\ 458 echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-updates main restricted universe multiverse" >> /etc/apt/sources.list && \\ 459 echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-security main restricted universe multiverse" >> /etc/apt/sources.list && \\ 460 echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-proposed main restricted universe multiverse" >> /etc/apt/sources.list && \\ 461 echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-backports main restricted universe multiverse" >> /etc/apt/sources.list 462""" 463 464# Special flags for proxying. 465proxy_cmd = "" 466proxy_args = [] 467if http_proxy: 468 proxy_cmd = f""" 469RUN echo "[http]" >> {homedir}/.gitconfig && \ 470 echo "proxy = {http_proxy}" >> {homedir}/.gitconfig 471""" 472 proxy_args.extend( 473 [ 474 "--build-arg", 475 f"http_proxy={http_proxy}", 476 "--build-arg", 477 "https_proxy={https_proxy}", 478 ] 479 ) 480 481# Create docker image that can run package unit tests 482dockerfile = f""" 483FROM {docker_base}{distro} as openbmc-base 484 485{mirror} 486 487ENV DEBIAN_FRONTEND noninteractive 488 489ENV PYTHONPATH "/usr/local/lib/python3.8/site-packages/" 490 491# We need the keys to be imported for dbgsym repos 492# New releases have a package, older ones fall back to manual fetching 493# https://wiki.ubuntu.com/Debug%20Symbol%20Packages 494RUN apt-get update && ( apt-get install ubuntu-dbgsym-keyring || ( apt-get install -yy dirmngr && \ 495 apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622 ) ) 496 497# Parse the current repo list into a debug repo list 498RUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list 499 500# Remove non-existent debug repos 501RUN sed -i '/-\(backports\|security\) /d' /etc/apt/sources.list.d/debug.list 502 503RUN cat /etc/apt/sources.list.d/debug.list 504 505RUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \ 506 gcc-10 \ 507 g++-10 \ 508 libc6-dbg \ 509 libc6-dev \ 510 libtool \ 511 bison \ 512 libdbus-1-dev \ 513 flex \ 514 cmake \ 515 python3 \ 516 python3-dev\ 517 python3-yaml \ 518 python3-mako \ 519 python3-pip \ 520 python3-setuptools \ 521 python3-git \ 522 python3-socks \ 523 pkg-config \ 524 autoconf \ 525 autoconf-archive \ 526 libsystemd-dev \ 527 systemd \ 528 libssl-dev \ 529 libevdev-dev \ 530 libevdev2-dbgsym \ 531 libjpeg-dev \ 532 libpng-dev \ 533 ninja-build \ 534 sudo \ 535 curl \ 536 git \ 537 dbus \ 538 iputils-ping \ 539 clang-10 \ 540 clang-format-10 \ 541 clang-tidy-10 \ 542 clang-tools-10 \ 543 shellcheck \ 544 npm \ 545 iproute2 \ 546 libnl-3-dev \ 547 libnl-genl-3-dev \ 548 libconfig++-dev \ 549 libsnmp-dev \ 550 valgrind \ 551 valgrind-dbg \ 552 libpam0g-dev \ 553 xxd \ 554 libi2c-dev \ 555 wget \ 556 libldap2-dev \ 557 libprotobuf-dev \ 558 libperlio-gzip-perl \ 559 libjson-perl \ 560 protobuf-compiler \ 561 libgpiod-dev \ 562 device-tree-compiler \ 563 cppcheck \ 564 libpciaccess-dev \ 565 libmimetic-dev \ 566 libxml2-utils \ 567 libxml-simple-perl 568 569RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 1000 \ 570 --slave /usr/bin/g++ g++ /usr/bin/g++-10 \ 571 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 \ 572 --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-10 \ 573 --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-10 574 575 576RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 1000 \ 577 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-10 \ 578 --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-10 \ 579 --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-10 \ 580 --slave /usr/bin/run-clang-tidy.py run-clang-tidy.py /usr/bin/run-clang-tidy-10.py 581 582RUN pip3 install inflection 583RUN pip3 install pycodestyle 584RUN pip3 install jsonschema 585RUN pip3 install meson==0.54.3 586RUN pip3 install protobuf 587 588{pkg_generate_packages()} 589 590# Build the final output image 591FROM openbmc-base 592{copy_cmds} 593 594# Some of our infrastructure still relies on the presence of this file 595# even though it is no longer needed to rebuild the docker environment 596# NOTE: The file is sorted to ensure the ordering is stable. 597RUN echo '{depcache}' > /tmp/depcache 598 599# Final configuration for the workspace 600RUN grep -q {gid} /etc/group || groupadd -g {gid} {username} 601RUN mkdir -p "{os.path.dirname(homedir)}" 602RUN grep -q {uid} /etc/passwd || useradd -d {homedir} -m -u {uid} -g {gid} {username} 603RUN sed -i '1iDefaults umask=000' /etc/sudoers 604RUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers 605 606{proxy_cmd} 607 608RUN /bin/bash 609""" 610 611# Do the docker build. 612for line in docker.build( 613 proxy_args, 614 "--network=host", 615 "-t", 616 docker_image_name, 617 "-", 618 _in=dockerfile, 619 _iter=True, 620): 621 print(line, end="", flush=True) 622