1#!/usr/bin/env python3 2# 3# Build the required docker image to run package unit tests 4# 5# Script Variables: 6# DOCKER_IMG_NAME: <optional, the name of the docker image to generate> 7# default is openbmc/ubuntu-unit-test 8# DISTRO: <optional, the distro to build a docker image against> 9# default is ubuntu:eoan 10# BRANCH: <optional, branch to build from each of the openbmc/ 11# repositories> 12# default is master, which will be used if input branch not 13# provided or not found 14# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the 15# default ones in /etc/apt/sources.list> 16# default is empty, and no mirror is used. 17# http_proxy The HTTP address of the proxy server to connect to. 18# Default: "", proxy is not setup if this is not set 19 20import os 21import sys 22from sh import docker, git, nproc, uname 23 24# Read a bunch of environment variables. 25docker_image_name = os.environ.get("DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test") 26distro = os.environ.get("DISTRO", "ubuntu:focal") 27branch = os.environ.get("BRANCH", "master") 28ubuntu_mirror = os.environ.get("UBUNTU_MIRROR") 29http_proxy = os.environ.get("http_proxy") 30prefix = "/usr/local" 31 32# Set up some common variables. 33proc_count = nproc().strip() 34username = os.environ.get("USER") 35homedir = os.environ.get("HOME") 36gid = os.getgid() 37uid = os.getuid() 38 39# Determine the architecture for Docker. 40arch = uname("-m").strip() 41if arch == "ppc64le": 42 docker_base = "ppc64le/" 43elif arch == "x86_64": 44 docker_base = "" 45else: 46 print(f"Unsupported system architecture({arch}) found for docker image") 47 sys.exit(1) 48 49# Packages to include in image. 50packages = { 51 "boost": { 52 "rev": "1.74.0", 53 "url": ( 54 lambda pkg, rev: f"https://dl.bintray.com/boostorg/release/{rev}/source/{pkg}_{rev.replace('.', '_')}.tar.bz2" 55 ), 56 "build_type": "custom", 57 "build_steps": [ 58 f"./bootstrap.sh --prefix={prefix} --with-libraries=context,coroutine", 59 "./b2", 60 f"./b2 install --prefix={prefix}", 61 ], 62 }, 63 "USCiLab/cereal": { 64 "rev": "v1.3.0", 65 "build_type": "custom", 66 "build_steps": [f"cp -a include/cereal/ {prefix}/include/"], 67 }, 68 "catchorg/Catch2": { 69 "rev": "v2.12.2", 70 "build_type": "cmake", 71 "config_flags": ["-DBUILD_TESTING=OFF", "-DCATCH_INSTALL_DOCS=OFF"], 72 }, 73 "CLIUtils/CLI11": { 74 "rev": "v1.9.0", 75 "build_type": "cmake", 76 "config_flags": [ 77 "-DBUILD_TESTING=OFF", 78 "-DCLI11_BUILD_DOCS=OFF", 79 "-DCLI11_BUILD_EXAMPLES=OFF", 80 ], 81 }, 82 "fmtlib/fmt": { 83 "rev": "6.2.1", 84 "build_type": "cmake", 85 "config_flags": [ 86 "-DFMT_DOC=OFF", 87 "-DFMT_TEST=OFF", 88 ], 89 }, 90 # Snapshot from 2020-01-03 91 "Naios/function2": { 92 "rev": "3a0746bf5f601dfed05330aefcb6854354fce07d", 93 "build_type": "custom", 94 "build_steps": [ 95 f"mkdir {prefix}/include/function2", 96 f"cp include/function2/function2.hpp {prefix}/include/function2/", 97 ], 98 }, 99 # Snapshot from 2020-02-13 100 "google/googletest": { 101 "rev": "23b2a3b1cf803999fb38175f6e9e038a4495c8a5", 102 "build_type": "cmake", 103 "config_env": ["CXXFLAGS=-std=c++17"], 104 "config_flags": ["-DTHREADS_PREFER_PTHREAD_FLAG=ON"], 105 }, 106 # Release 2020-08-06 107 "nlohmann/json": { 108 "rev": "v3.9.1", 109 "build_type": "custom", 110 "build_steps": [ 111 f"mkdir {prefix}/include/nlohmann", 112 f"cp include/nlohmann/json.hpp {prefix}/include/nlohmann", 113 f"ln -s {prefix}/include/nlohmann/json.hpp {prefix}/include/json.hpp", 114 ], 115 }, 116 # Snapshot from 2019-05-24 117 "linux-test-project/lcov": { 118 "rev": "75fbae1cfc5027f818a0bb865bf6f96fab3202da", 119 "build_type": "make", 120 }, 121 # dev-5.0 2019-05-03 122 "openbmc/linux": { 123 "rev": "8bf6567e77f7aa68975b7c9c6d044bba690bf327", 124 "build_type": "custom", 125 "build_steps": [ 126 f"make -j{proc_count} defconfig", 127 f"make INSTALL_HDR_PATH={prefix} headers_install", 128 ], 129 }, 130 # Snapshot from 2019-09-03 131 "LibVNC/libvncserver": { 132 "rev": "1354f7f1bb6962dab209eddb9d6aac1f03408110", 133 "build_type": "cmake", 134 }, 135 "martinmoene/span-lite": { 136 "rev": "v0.7.0", 137 "build_type": "cmake", 138 "config_flags": [ 139 "-DSPAN_LITE_OPT_BUILD_TESTS=OFF", 140 ], 141 }, 142 # version from meta-openembedded/meta-oe/recipes-support/libtinyxml2/libtinyxml2_5.0.1.bb 143 "leethomason/tinyxml2": { 144 "rev": "37bc3aca429f0164adf68c23444540b4a24b5778", 145 "build_type": "cmake", 146 }, 147 # version from /meta-openembedded/meta-oe/recipes-devtools/boost-url/boost-url_git.bb 148 "CPPAlliance/url": { 149 "rev": "a56ae0df6d3078319755fbaa67822b4fa7fd352b", 150 "build_type": "cmake", 151 "config_flags": [ 152 "-DBOOST_URL_BUILD_EXAMPLES=OFF", 153 "-DBOOST_URL_BUILD_TESTS=OFF", 154 "-DBOOST_URL_STANDALONE=ON", 155 ], 156 }, 157 # version from meta-openembedded/meta-oe/recipes-devtools/valijson/valijson_git.bb 158 "tristanpenman/valijson": { 159 "rev": "c2f22fddf599d04dc33fcd7ed257c698a05345d9", 160 "build_type": "cmake", 161 "config_flags": [ 162 "-DBUILD_TESTS=0", 163 "-DINSTALL_HEADERS=1", 164 ], 165 }, 166 # version from meta-openembedded/meta-oe/recipes-devtools/nlohmann-fifo/nlohmann-fifo_git.bb 167 "nlohmann/fifo_map": { 168 "rev": "0dfbf5dacbb15a32c43f912a7e66a54aae39d0f9", 169 "build_type": "custom", 170 "build_steps": [f"cp src/fifo_map.hpp {prefix}/include/"], 171 }, 172 "open-power/pdbg": {"build_type": "autoconf"}, 173 "openbmc/gpioplus": { 174 "depends": ["openbmc/stdplus"], 175 "build_type": "meson", 176 "config_flags": [ 177 "-Dexamples=false", 178 "-Dtests=disabled", 179 ], 180 }, 181 "openbmc/phosphor-dbus-interfaces": { 182 "depends": ["openbmc/sdbusplus"], 183 "build_type": "meson", 184 "config_flags": [ 185 "-Ddata_com_ibm=true", 186 "-Ddata_org_open_power=true", 187 ], 188 }, 189 "openbmc/phosphor-logging": { 190 "depends": [ 191 "USCiLab/cereal", 192 "nlohmann/fifo_map", 193 "openbmc/phosphor-dbus-interfaces", 194 "openbmc/sdbusplus", 195 "openbmc/sdeventplus", 196 ], 197 "build_type": "autoconf", 198 "config_flags": [ 199 "--enable-metadata-processing", 200 f"YAML_DIR={prefix}/share/phosphor-dbus-yaml/yaml", 201 ], 202 }, 203 "openbmc/phosphor-objmgr": { 204 "depends": [ 205 "boost", 206 "leethomason/tinyxml2", 207 "openbmc/phosphor-logging", 208 "openbmc/sdbusplus", 209 ], 210 "build_type": "autoconf", 211 }, 212 "openbmc/pldm": { 213 "depends": [ 214 "CLIUtils/CLI11", 215 "boost", 216 "nlohmann/json", 217 "openbmc/phosphor-dbus-interfaces", 218 "openbmc/phosphor-logging", 219 "openbmc/sdbusplus", 220 "openbmc/sdeventplus", 221 ], 222 "build_type": "meson", 223 "config_flags": [ 224 "-Dlibpldm-only=enabled", 225 "-Doem-ibm=enabled", 226 "-Dtests=disabled", 227 ], 228 }, 229 "openbmc/sdbusplus": { 230 "build_type": "meson", 231 "custom_post_dl": [ 232 "cd tools", 233 f"./setup.py install --root=/ --prefix={prefix}", 234 "cd ..", 235 ], 236 "config_flags": [ 237 "-Dexamples=disabled", 238 "-Dtests=disabled", 239 ], 240 }, 241 "openbmc/sdeventplus": { 242 "depends": ["Naios/function2", "openbmc/stdplus"], 243 "build_type": "meson", 244 "config_flags": [ 245 "-Dexamples=false", 246 "-Dtests=disabled", 247 ], 248 }, 249 "openbmc/stdplus": { 250 "depends": ["fmtlib/fmt", "martinmoene/span-lite"], 251 "build_type": "meson", 252 "config_flags": [ 253 "-Dexamples=false", 254 "-Dtests=disabled", 255 ], 256 }, 257} 258 259 260def pkg_rev(pkg): 261 return packages[pkg]["rev"] 262 263 264def pkg_stagename(pkg): 265 if not pkg.startswith("openbmc/"): 266 pkg = "openbmc/" + pkg 267 return pkg.replace("/", "-") 268 269 270def pkg_url(pkg): 271 if "url" in packages[pkg]: 272 return packages[pkg]["url"](pkg, pkg_rev(pkg)) 273 return f"https://github.com/{pkg}/archive/{pkg_rev(pkg)}.tar.gz" 274 275 276def pkg_download(pkg): 277 url = pkg_url(pkg) 278 if ".tar." not in url: 279 raise NotImplementedError(f"Unhandled download type for {pkg}: {url}") 280 cmd = f"curl -L {url} | tar -x" 281 if url.endswith(".bz2"): 282 cmd += "j" 283 if url.endswith(".gz"): 284 cmd += "z" 285 return cmd 286 287 288def pkg_copycmds(pkg=None): 289 pkgs = [] 290 if pkg: 291 if "depends" not in packages[pkg]: 292 return "" 293 pkgs = sorted(packages[pkg]["depends"]) 294 else: 295 pkgs = sorted(packages.keys()) 296 297 copy_cmds = "" 298 for p in pkgs: 299 copy_cmds += f"COPY --from={pkg_stagename(p)} {prefix} {prefix}\n" 300 # Workaround for upstream docker bug and multiple COPY cmds 301 # https://github.com/moby/moby/issues/37965 302 copy_cmds += "RUN true\n" 303 return copy_cmds 304 305 306def pkg_cd_srcdir(pkg): 307 return f"cd {pkg.split('/')[-1]}* && " 308 309 310def pkg_build(pkg): 311 result = f"RUN {pkg_download(pkg)} && " 312 result += pkg_cd_srcdir(pkg) 313 314 if "custom_post_dl" in packages[pkg]: 315 result += " && ".join(packages[pkg]["custom_post_dl"]) + " && " 316 317 build_type = packages[pkg]["build_type"] 318 if build_type == "autoconf": 319 result += pkg_build_autoconf(pkg) 320 elif build_type == "cmake": 321 result += pkg_build_cmake(pkg) 322 elif build_type == "custom": 323 result += pkg_build_custom(pkg) 324 elif build_type == "make": 325 result += pkg_build_make(pkg) 326 elif build_type == "meson": 327 result += pkg_build_meson(pkg) 328 else: 329 raise NotImplementedError( 330 f"Unhandled build type for {pkg}: {packages[pkg]['build_type']}" 331 ) 332 333 return result 334 335 336def pkg_build_autoconf(pkg): 337 options = " ".join(packages[pkg].get("config_flags", [])) 338 env = " ".join(packages[pkg].get("config_env", [])) 339 result = "./bootstrap.sh && " 340 result += f"{env} ./configure {configure_flags} {options} && " 341 result += f"make -j{proc_count} && " 342 result += "make install " 343 return result 344 345 346def pkg_build_cmake(pkg): 347 options = " ".join(packages[pkg].get("config_flags", [])) 348 env = " ".join(packages[pkg].get("config_env", [])) 349 result = "mkdir builddir && cd builddir && " 350 result += f"{env} cmake {cmake_flags} {options} .. && " 351 result += f"make -j{proc_count} && " 352 result += f"make install &&" 353 result += "cd .. " 354 return result 355 356 357def pkg_build_custom(pkg): 358 return " && ".join(packages[pkg].get("build_steps", [])) 359 360 361def pkg_build_make(pkg): 362 result = f"make -j{proc_count} && " 363 result += "make install " 364 return result 365 366 367def pkg_build_meson(pkg): 368 options = " ".join(packages[pkg].get("config_flags", [])) 369 env = " ".join(packages[pkg].get("config_env", [])) 370 result = f"{env} meson builddir {meson_flags} {options} && " 371 result += "ninja -C builddir && ninja -C builddir install " 372 return result 373 374 375def pkg_generate(pkg): 376 if "__pkg_built" in packages[pkg]: 377 return None 378 379 result = "" 380 381 for deppkg in sorted(packages[pkg].get("depends", [])): 382 dep_build = pkg_generate(deppkg) 383 if not dep_build: 384 continue 385 result += dep_build + "\n" 386 387 result += f"FROM openbmc-base as {pkg_stagename(pkg)}\n" 388 result += pkg_copycmds(pkg) 389 result += pkg_build(pkg) 390 391 packages[pkg]["__pkg_built"] = True 392 return result 393 394 395def pkg_generate_packages(): 396 result = "" 397 for pkg in sorted(packages.keys()): 398 build = pkg_generate(pkg) 399 if not build: 400 continue 401 result += build + "\n" 402 return result 403 404 405# Look up the HEAD for missing a static rev. 406pkg_lookups = {} 407for pkg in packages.keys(): 408 if "rev" in packages[pkg]: 409 continue 410 pkg_lookups[pkg] = git( 411 "ls-remote", "--heads", f"https://github.com/{pkg}", _bg=True 412 ) 413for pkg, result in pkg_lookups.items(): 414 for line in result.stdout.decode().split("\n"): 415 if f"refs/heads/{branch}" in line: 416 packages[pkg]["rev"] = line.strip().split()[0] 417 elif "refs/heads/master" in line and p not in packages: 418 packages[pkg]["rev"] = line.strip().split()[0] 419 420# Create the contents of the '/tmp/depcache'. 421# This needs to be sorted for consistency. 422depcache = "" 423for pkg in sorted(packages.keys()): 424 depcache += "%s:%s," % (pkg, pkg_rev(pkg)) 425 426# Define common flags used for builds 427configure_flags = " ".join( 428 [ 429 f"--prefix={prefix}", 430 ] 431) 432cmake_flags = " ".join( 433 [ 434 "-DCMAKE_BUILD_TYPE=RelWithDebInfo", 435 "-DBUILD_SHARED_LIBS=ON", 436 f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}", 437 ] 438) 439meson_flags = " ".join( 440 [ 441 "--wrap-mode=nodownload", 442 f"-Dprefix={prefix}", 443 ] 444) 445 446# Build the commands needed to compose our final image 447# We must sort the packages, otherwise we might produce an unstable 448# docker file and rebuild the image unnecessarily 449copy_cmds = pkg_copycmds() 450 451# Special flags if setting up a deb mirror. 452mirror = "" 453if "ubuntu" in distro and ubuntu_mirror: 454 mirror = f""" 455RUN echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME) main restricted universe multiverse" > /etc/apt/sources.list && \\ 456 echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-updates main restricted universe multiverse" >> /etc/apt/sources.list && \\ 457 echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-security main restricted universe multiverse" >> /etc/apt/sources.list && \\ 458 echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-proposed main restricted universe multiverse" >> /etc/apt/sources.list && \\ 459 echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-backports main restricted universe multiverse" >> /etc/apt/sources.list 460""" 461 462# Special flags for proxying. 463proxy_cmd = "" 464proxy_args = [] 465if http_proxy: 466 proxy_cmd = f""" 467RUN echo "[http]" >> {homedir}/.gitconfig && \ 468 echo "proxy = {http_proxy}" >> {homedir}/.gitconfig 469""" 470 proxy_args.extend( 471 [ 472 "--build-arg", 473 f"http_proxy={http_proxy}", 474 "--build-arg", 475 "https_proxy={https_proxy}", 476 ] 477 ) 478 479# Create docker image that can run package unit tests 480dockerfile = f""" 481FROM {docker_base}{distro} as openbmc-base 482 483{mirror} 484 485ENV DEBIAN_FRONTEND noninteractive 486 487ENV PYTHONPATH "/usr/local/lib/python3.8/site-packages/" 488 489# We need the keys to be imported for dbgsym repos 490# New releases have a package, older ones fall back to manual fetching 491# https://wiki.ubuntu.com/Debug%20Symbol%20Packages 492RUN apt-get update && ( apt-get install ubuntu-dbgsym-keyring || ( apt-get install -yy dirmngr && \ 493 apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622 ) ) 494 495# Parse the current repo list into a debug repo list 496RUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list 497 498# Remove non-existent debug repos 499RUN sed -i '/-\(backports\|security\) /d' /etc/apt/sources.list.d/debug.list 500 501RUN cat /etc/apt/sources.list.d/debug.list 502 503RUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \ 504 gcc-10 \ 505 g++-10 \ 506 libc6-dbg \ 507 libc6-dev \ 508 libtool \ 509 bison \ 510 libdbus-1-dev \ 511 flex \ 512 cmake \ 513 python3 \ 514 python3-dev\ 515 python3-yaml \ 516 python3-mako \ 517 python3-pip \ 518 python3-setuptools \ 519 python3-git \ 520 python3-socks \ 521 pkg-config \ 522 autoconf \ 523 autoconf-archive \ 524 libsystemd-dev \ 525 systemd \ 526 libssl-dev \ 527 libevdev-dev \ 528 libevdev2-dbgsym \ 529 libjpeg-dev \ 530 libpng-dev \ 531 ninja-build \ 532 sudo \ 533 curl \ 534 git \ 535 dbus \ 536 iputils-ping \ 537 clang-10 \ 538 clang-format-10 \ 539 clang-tidy-10 \ 540 clang-tools-10 \ 541 shellcheck \ 542 npm \ 543 iproute2 \ 544 libnl-3-dev \ 545 libnl-genl-3-dev \ 546 libconfig++-dev \ 547 libsnmp-dev \ 548 valgrind \ 549 valgrind-dbg \ 550 libpam0g-dev \ 551 xxd \ 552 libi2c-dev \ 553 wget \ 554 libldap2-dev \ 555 libprotobuf-dev \ 556 libperlio-gzip-perl \ 557 libjson-perl \ 558 protobuf-compiler \ 559 libgpiod-dev \ 560 device-tree-compiler \ 561 cppcheck \ 562 libpciaccess-dev \ 563 libmimetic-dev \ 564 libxml2-utils \ 565 libxml-simple-perl 566 567RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 1000 \ 568 --slave /usr/bin/g++ g++ /usr/bin/g++-10 \ 569 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 \ 570 --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-10 \ 571 --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-10 572 573 574RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 1000 \ 575 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-10 \ 576 --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-10 \ 577 --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-10 \ 578 --slave /usr/bin/run-clang-tidy.py run-clang-tidy.py /usr/bin/run-clang-tidy-10.py 579 580RUN pip3 install inflection 581RUN pip3 install pycodestyle 582RUN pip3 install jsonschema 583RUN pip3 install meson==0.54.3 584RUN pip3 install protobuf 585 586{pkg_generate_packages()} 587 588# Build the final output image 589FROM openbmc-base 590{copy_cmds} 591 592# Some of our infrastructure still relies on the presence of this file 593# even though it is no longer needed to rebuild the docker environment 594# NOTE: The file is sorted to ensure the ordering is stable. 595RUN echo '{depcache}' > /tmp/depcache 596 597# Final configuration for the workspace 598RUN grep -q {gid} /etc/group || groupadd -g {gid} {username} 599RUN mkdir -p "{os.path.dirname(homedir)}" 600RUN grep -q {uid} /etc/passwd || useradd -d {homedir} -m -u {uid} -g {gid} {username} 601RUN sed -i '1iDefaults umask=000' /etc/sudoers 602RUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers 603 604{proxy_cmd} 605 606RUN /bin/bash 607""" 608 609# Do the docker build. 610for line in docker.build( 611 proxy_args, 612 "--network=host", 613 "-t", 614 docker_image_name, 615 "-", 616 _in=dockerfile, 617 _iter=True, 618): 619 print(line, end="", flush=True) 620