1#!/usr/bin/env python3 2# 3# Build the required docker image to run package unit tests 4# 5# Script Variables: 6# DOCKER_IMG_NAME: <optional, the name of the docker image to generate> 7# default is openbmc/ubuntu-unit-test 8# DISTRO: <optional, the distro to build a docker image against> 9# default is ubuntu:eoan 10# BRANCH: <optional, branch to build from each of the openbmc/ 11# repositories> 12# default is master, which will be used if input branch not 13# provided or not found 14# UBUNTU_MIRROR: <optional, the URL of a mirror of Ubuntu to override the 15# default ones in /etc/apt/sources.list> 16# default is empty, and no mirror is used. 17# http_proxy The HTTP address of the proxy server to connect to. 18# Default: "", proxy is not setup if this is not set 19 20import os 21import sys 22from sh import docker, git, nproc, uname 23 24# Read a bunch of environment variables. 25docker_image_name = os.environ.get("DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test") 26distro = os.environ.get("DISTRO", "ubuntu:focal") 27branch = os.environ.get("BRANCH", "master") 28ubuntu_mirror = os.environ.get("UBUNTU_MIRROR") 29http_proxy = os.environ.get("http_proxy") 30 31# Set up some common variables. 32proc_count = nproc().strip() 33username = os.environ.get("USER") 34homedir = os.environ.get("HOME") 35gid = os.getgid() 36uid = os.getuid() 37 38# Determine the architecture for Docker. 39arch = uname("-m").strip() 40if arch == "ppc64le": 41 docker_base = "ppc64le/" 42elif arch == "x86_64": 43 docker_base = "" 44else: 45 print(f"Unsupported system architecture({arch}) found for docker image") 46 sys.exit(1) 47 48# Packages to include in image. 49packages = { 50 "boost": {"rev": "1.74.0"}, 51 "cereal": {"rev": "v1.3.0"}, 52 "catch2": {"rev": "v2.12.2"}, 53 "CLI11": {"rev": "v1.9.0"}, 54 "fmt": {"rev": "6.2.1"}, 55 # Snapshot from 2020-01-03 56 "function2": {"rev": "3a0746bf5f601dfed05330aefcb6854354fce07d"}, 57 # Snapshot from 2020-02-13 58 "googletest": {"rev": "23b2a3b1cf803999fb38175f6e9e038a4495c8a5"}, 59 # Release 2020-08-06 60 "json": {"rev": "v3.9.1"}, 61 # Snapshot from 2019-05-24 62 "lcov": {"rev": "75fbae1cfc5027f818a0bb865bf6f96fab3202da"}, 63 # dev-5.0 2019-05-03 64 "linux-headers": {"rev": "8bf6567e77f7aa68975b7c9c6d044bba690bf327"}, 65 # Snapshot from 2019-09-03 66 "libvncserver": {"rev": "1354f7f1bb6962dab209eddb9d6aac1f03408110"}, 67 "span-lite": {"rev": "v0.7.0"}, 68 # version from meta-openembedded/meta-oe/recipes-support/libtinyxml2/libtinyxml2_5.0.1.bb 69 "tinyxml2": {"rev": "37bc3aca429f0164adf68c23444540b4a24b5778"}, 70 # version from /meta-openembedded/meta-oe/recipes-devtools/boost-url/boost-url_git.bb 71 "boost-url": {"rev": "a56ae0df6d3078319755fbaa67822b4fa7fd352b"}, 72 # version from meta-openembedded/meta-oe/recipes-devtools/valijson/valijson_git.bb 73 "valijson": {"rev": "c2f22fddf599d04dc33fcd7ed257c698a05345d9"}, 74 # version from meta-openembedded/meta-oe/recipes-devtools/nlohmann-fifo/nlohmann-fifo_git.bb 75 "fifo_map": {"rev": "0dfbf5dacbb15a32c43f912a7e66a54aae39d0f9"}, 76 "open-power/pdbg": {}, 77 "openbmc/gpioplus": {}, 78 "openbmc/phosphor-dbus-interfaces": {}, 79 "openbmc/phosphor-logging": {}, 80 "openbmc/phosphor-objmgr": {}, 81 "openbmc/pldm": {}, 82 "openbmc/sdbusplus": {}, 83 "openbmc/sdeventplus": {}, 84 "openbmc/stdplus": {}, 85} 86 87 88def pkg_rev(pkg): 89 return packages[pkg]["rev"] 90 91 92# Look up the HEAD for missing a static rev. 93pkg_lookups = {} 94for pkg in packages.keys(): 95 if "rev" in packages[pkg]: 96 continue 97 pkg_lookups[pkg] = git( 98 "ls-remote", "--heads", f"https://github.com/{pkg}", _bg=True 99 ) 100for pkg, result in pkg_lookups.items(): 101 for line in result.stdout.decode().split("\n"): 102 if f"refs/heads/{branch}" in line: 103 packages[pkg]["rev"] = line.strip().split()[0] 104 elif "refs/heads/master" in line and p not in packages: 105 packages[pkg]["rev"] = line.strip().split()[0] 106 107# Create the contents of the '/tmp/depcache'. 108# This needs to be sorted for consistency. 109depcache = "" 110for pkg in sorted(packages.keys()): 111 depcache += "%s:%s," % (pkg, pkg_rev(pkg)) 112 113# Define common flags used for builds 114prefix = "/usr/local" 115configure_flags = " ".join( 116 [ 117 f"--prefix={prefix}", 118 ] 119) 120cmake_flags = " ".join( 121 [ 122 "-DCMAKE_BUILD_TYPE=RelWithDebInfo", 123 "-DBUILD_SHARED_LIBS=ON", 124 f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}", 125 ] 126) 127meson_flags = " ".join( 128 [ 129 "--wrap-mode=nodownload", 130 f"-Dprefix={prefix}", 131 ] 132) 133 134 135def stagename(name): 136 if not name.startswith("openbmc/"): 137 name = "openbmc/" + name 138 return name.replace("/", "-") 139 140 141# Build the commands needed to compose our final image 142# We must sort the packages, otherwise we might produce an unstable 143# docker file and rebuild the image unnecessarily 144copy_cmds = "" 145for pkg in sorted(packages.keys()): 146 copy_cmds += f"COPY --from={stagename(pkg)} {prefix} {prefix}\n" 147 # Workaround for upstream docker bug and multiple COPY cmds 148 # https://github.com/moby/moby/issues/37965 149 copy_cmds += "RUN true\n" 150 151# Special flags if setting up a deb mirror. 152mirror = "" 153if "ubuntu" in distro and ubuntu_mirror: 154 mirror = f""" 155RUN echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME) main restricted universe multiverse" > /etc/apt/sources.list && \\ 156 echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-updates main restricted universe multiverse" >> /etc/apt/sources.list && \\ 157 echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-security main restricted universe multiverse" >> /etc/apt/sources.list && \\ 158 echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-proposed main restricted universe multiverse" >> /etc/apt/sources.list && \\ 159 echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-backports main restricted universe multiverse" >> /etc/apt/sources.list 160""" 161 162# Special flags for proxying. 163proxy_cmd = "" 164proxy_args = [] 165if http_proxy: 166 proxy_cmd = f""" 167RUN echo "[http]" >> {homedir}/.gitconfig && \ 168 echo "proxy = {http_proxy}" >> {homedir}/.gitconfig 169""" 170 proxy_args.extend( 171 [ 172 "--build-arg", 173 f"http_proxy={http_proxy}", 174 "--build-arg", 175 "https_proxy={https_proxy}", 176 ] 177 ) 178 179# Create docker image that can run package unit tests 180dockerfile = f""" 181FROM {docker_base}{distro} as openbmc-base 182 183{mirror} 184 185ENV DEBIAN_FRONTEND noninteractive 186 187ENV PYTHONPATH "/usr/local/lib/python3.8/site-packages/" 188 189# We need the keys to be imported for dbgsym repos 190# New releases have a package, older ones fall back to manual fetching 191# https://wiki.ubuntu.com/Debug%20Symbol%20Packages 192RUN apt-get update && ( apt-get install ubuntu-dbgsym-keyring || ( apt-get install -yy dirmngr && \ 193 apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622 ) ) 194 195# Parse the current repo list into a debug repo list 196RUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list 197 198# Remove non-existent debug repos 199RUN sed -i '/-\(backports\|security\) /d' /etc/apt/sources.list.d/debug.list 200 201RUN cat /etc/apt/sources.list.d/debug.list 202 203RUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \ 204 gcc-10 \ 205 g++-10 \ 206 libc6-dbg \ 207 libc6-dev \ 208 libtool \ 209 bison \ 210 libdbus-1-dev \ 211 flex \ 212 cmake \ 213 python3 \ 214 python3-dev\ 215 python3-yaml \ 216 python3-mako \ 217 python3-pip \ 218 python3-setuptools \ 219 python3-git \ 220 python3-socks \ 221 pkg-config \ 222 autoconf \ 223 autoconf-archive \ 224 libsystemd-dev \ 225 systemd \ 226 libssl-dev \ 227 libevdev-dev \ 228 libevdev2-dbgsym \ 229 libjpeg-dev \ 230 libpng-dev \ 231 ninja-build \ 232 sudo \ 233 curl \ 234 git \ 235 dbus \ 236 iputils-ping \ 237 clang-10 \ 238 clang-format-10 \ 239 clang-tidy-10 \ 240 clang-tools-10 \ 241 shellcheck \ 242 npm \ 243 iproute2 \ 244 libnl-3-dev \ 245 libnl-genl-3-dev \ 246 libconfig++-dev \ 247 libsnmp-dev \ 248 valgrind \ 249 valgrind-dbg \ 250 libpam0g-dev \ 251 xxd \ 252 libi2c-dev \ 253 wget \ 254 libldap2-dev \ 255 libprotobuf-dev \ 256 libperlio-gzip-perl \ 257 libjson-perl \ 258 protobuf-compiler \ 259 libgpiod-dev \ 260 device-tree-compiler \ 261 cppcheck \ 262 libpciaccess-dev \ 263 libmimetic-dev \ 264 libxml2-utils \ 265 libxml-simple-perl 266 267RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 1000 \ 268 --slave /usr/bin/g++ g++ /usr/bin/g++-10 \ 269 --slave /usr/bin/gcov gcov /usr/bin/gcov-10 \ 270 --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-10 \ 271 --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-10 272 273 274RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 1000 \ 275 --slave /usr/bin/clang++ clang++ /usr/bin/clang++-10 \ 276 --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-10 \ 277 --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-10 \ 278 --slave /usr/bin/run-clang-tidy.py run-clang-tidy.py /usr/bin/run-clang-tidy-10.py 279 280RUN pip3 install inflection 281RUN pip3 install pycodestyle 282RUN pip3 install jsonschema 283RUN pip3 install meson==0.54.3 284RUN pip3 install protobuf 285 286FROM openbmc-base as openbmc-lcov 287RUN curl -L https://github.com/linux-test-project/lcov/archive/{pkg_rev('lcov')}.tar.gz | tar -xz && \ 288cd lcov-* && \ 289make -j{proc_count} && \ 290make install 291 292FROM openbmc-base as openbmc-function2 293RUN mkdir {prefix}/include/function2 && \ 294curl -L -o {prefix}/include/function2/function2.hpp https://raw.githubusercontent.com/Naios/function2/{pkg_rev('function2')}/include/function2/function2.hpp 295 296FROM openbmc-base as openbmc-googletest 297RUN curl -L https://github.com/google/googletest/archive/{pkg_rev('googletest')}.tar.gz | tar -xz && \ 298cd googletest-* && \ 299mkdir build && \ 300cd build && \ 301CXXFLAGS=-std=c++17 cmake {cmake_flags} -DTHREADS_PREFER_PTHREAD_FLAG=ON .. && \ 302make -j{proc_count} && \ 303make install 304 305FROM openbmc-base as openbmc-catch2 306RUN curl -L https://github.com/catchorg/Catch2/archive/{pkg_rev('catch2')}.tar.gz | tar -xz && \ 307cd Catch2-* && \ 308mkdir build && \ 309cd build && \ 310cmake {cmake_flags} -DBUILD_TESTING=OFF -DCATCH_INSTALL_DOCS=OFF .. && \ 311make -j{proc_count} && \ 312make install 313 314FROM openbmc-base as openbmc-cereal 315RUN curl -L https://github.com/USCiLab/cereal/archive/{pkg_rev('cereal')}.tar.gz | tar -xz && \ 316cp -a cereal-*/include/cereal/ {prefix}/include/ 317 318FROM openbmc-base as openbmc-CLI11 319RUN curl -L https://github.com/CLIUtils/CLI11/archive/{pkg_rev('CLI11')}.tar.gz | tar -xz && \ 320cd CLI11-* && \ 321mkdir build && \ 322cd build && \ 323cmake {cmake_flags} -DCLI11_BUILD_DOCS=OFF -DBUILD_TESTING=OFF -DCLI11_BUILD_EXAMPLES=OFF .. && \ 324make -j{proc_count} && \ 325make install 326 327FROM openbmc-base as openbmc-fmt 328RUN curl -L https://github.com/fmtlib/fmt/archive/{pkg_rev('fmt')}.tar.gz | tar -xz && \ 329cd fmt-* && \ 330mkdir build && \ 331cd build && \ 332cmake {cmake_flags} -DFMT_DOC=OFF -DFMT_TEST=OFF .. && \ 333make -j{proc_count} && \ 334make install 335 336FROM openbmc-base as openbmc-json 337RUN mkdir {prefix}/include/nlohmann/ && \ 338curl -L -o {prefix}/include/nlohmann/json.hpp https://github.com/nlohmann/json/releases/download/{pkg_rev('json')}/json.hpp && \ 339ln -s nlohmann/json.hpp {prefix}/include/json.hpp 340 341FROM openbmc-base as openbmc-fifo_map 342RUN curl -L https://github.com/nlohmann/fifo_map/archive/{pkg_rev('fifo_map')}.tar.gz | tar -xz && \ 343cd fifo_map-*/src && cp fifo_map.hpp {prefix}/include/ 344 345FROM openbmc-base as openbmc-span-lite 346RUN curl -L https://github.com/martinmoene/span-lite/archive/{pkg_rev('span-lite')}.tar.gz | tar -xz && \ 347cd span-lite-* && \ 348mkdir build && \ 349cd build && \ 350cmake {cmake_flags} -DSPAN_LITE_OPT_BUILD_TESTS=OFF .. && \ 351make -j{proc_count} && \ 352make install 353 354FROM openbmc-base as openbmc-linux-headers 355RUN curl -L https://github.com/openbmc/linux/archive/{pkg_rev('linux-headers')}.tar.gz | tar -xz && \ 356cd linux-* && \ 357make -j{proc_count} defconfig && \ 358make INSTALL_HDR_PATH=/usr/local headers_install 359 360FROM openbmc-base as openbmc-boost 361RUN curl -L https://dl.bintray.com/boostorg/release/{pkg_rev('boost')}/source/boost_$(echo "{pkg_rev('boost')}" | tr '.' '_').tar.bz2 | tar -xj && \ 362cd boost_*/ && \ 363./bootstrap.sh --prefix={prefix} --with-libraries=context,coroutine && \ 364./b2 && ./b2 install --prefix={prefix} 365 366FROM openbmc-base as openbmc-tinyxml2 367RUN curl -L https://github.com/leethomason/tinyxml2/archive/{pkg_rev('tinyxml2')}.tar.gz | tar -xz && \ 368cd tinyxml2-* && \ 369mkdir build && \ 370cd build && \ 371cmake {cmake_flags} .. && \ 372make -j{proc_count} && \ 373make install 374 375FROM openbmc-base as openbmc-boost-url 376RUN curl -L https://github.com/CPPAlliance/url/archive/{pkg_rev('boost-url')}.tar.gz | tar -xz && \ 377cd url-* && \ 378mkdir buildir && \ 379cd buildir && \ 380cmake {cmake_flags} -DBOOST_URL_STANDALONE=ON -DBOOST_URL_BUILD_TESTS=OFF -DBOOST_URL_BUILD_EXAMPLES=OFF .. && \ 381make -j{proc_count} && \ 382make install 383 384FROM openbmc-base as openbmc-valijson 385RUN curl -L https://github.com/tristanpenman/valijson/archive/{pkg_rev('valijson')}.tar.gz | tar -xz && \ 386cd valijson-* && \ 387mkdir build && \ 388cd build && \ 389cmake {cmake_flags} -DINSTALL_HEADERS=1 -DBUILD_TESTS=0 .. && \ 390make -j{proc_count} && \ 391make install 392 393FROM openbmc-base as openbmc-libvncserver 394RUN curl -L https://github.com/LibVNC/libvncserver/archive/{pkg_rev('libvncserver')}.tar.gz | tar -xz && \ 395cd libvncserver-* && \ 396mkdir build && \ 397cd build && \ 398cmake {cmake_flags} .. && \ 399make -j{proc_count} && \ 400make install 401 402FROM openbmc-base as openbmc-stdplus 403COPY --from=openbmc-fmt {prefix} {prefix} 404COPY --from=openbmc-span-lite {prefix} {prefix} 405RUN curl -L https://github.com/openbmc/stdplus/archive/{pkg_rev('openbmc/stdplus')}.tar.gz | tar -xz && \ 406cd stdplus-* && \ 407meson build {meson_flags} -Dtests=disabled -Dexamples=false && \ 408ninja -C build && \ 409ninja -C build install 410 411FROM openbmc-base as openbmc-sdbusplus 412RUN curl -L https://github.com/openbmc/sdbusplus/archive/{pkg_rev('openbmc/sdbusplus')}.tar.gz | tar -xz && \ 413cd sdbusplus-* && \ 414cd tools && ./setup.py install --root=/ --prefix={prefix} && \ 415cd .. && meson build {meson_flags} -Dtests=disabled -Dexamples=disabled && \ 416ninja -C build && \ 417ninja -C build install 418 419FROM openbmc-base as openbmc-sdeventplus 420COPY --from=openbmc-function2 {prefix} {prefix} 421COPY --from=openbmc-stdplus {prefix} {prefix} 422RUN curl -L https://github.com/openbmc/sdeventplus/archive/{pkg_rev('openbmc/sdeventplus')}.tar.gz | tar -xz && \ 423cd sdeventplus-* && \ 424meson build {meson_flags} -Dtests=disabled -Dexamples=false && \ 425ninja -C build && \ 426ninja -C build install 427 428FROM openbmc-base as openbmc-gpioplus 429COPY --from=openbmc-stdplus {prefix} {prefix} 430RUN curl -L https://github.com/openbmc/gpioplus/archive/{pkg_rev('openbmc/gpioplus')}.tar.gz | tar -xz && \ 431cd gpioplus-* && \ 432meson build {meson_flags} -Dtests=disabled -Dexamples=false && \ 433ninja -C build && \ 434ninja -C build install 435 436FROM openbmc-base as openbmc-phosphor-dbus-interfaces 437COPY --from=openbmc-sdbusplus {prefix} {prefix} 438RUN curl -L https://github.com/openbmc/phosphor-dbus-interfaces/archive/{pkg_rev('openbmc/phosphor-dbus-interfaces')}.tar.gz | tar -xz && \ 439cd phosphor-dbus-interfaces-* && \ 440meson build {meson_flags} -Ddata_org_open_power=true -Ddata_com_ibm=true && \ 441ninja -C build && \ 442ninja -C build install 443 444FROM openbmc-base as openbmc-phosphor-logging 445COPY --from=openbmc-cereal {prefix} {prefix} 446COPY --from=openbmc-sdbusplus {prefix} {prefix} 447COPY --from=openbmc-sdeventplus {prefix} {prefix} 448COPY --from=openbmc-phosphor-dbus-interfaces {prefix} {prefix} 449COPY --from=openbmc-fifo_map {prefix} {prefix} 450RUN curl -L https://github.com/openbmc/phosphor-logging/archive/{pkg_rev('openbmc/phosphor-logging')}.tar.gz | tar -xz && \ 451cd phosphor-logging-* && \ 452./bootstrap.sh && \ 453./configure {configure_flags} --enable-metadata-processing YAML_DIR={prefix}/share/phosphor-dbus-yaml/yaml && \ 454make -j{proc_count} && \ 455make install 456 457FROM openbmc-base as openbmc-phosphor-objmgr 458COPY --from=openbmc-boost {prefix} {prefix} 459COPY --from=openbmc-sdbusplus {prefix} {prefix} 460COPY --from=openbmc-tinyxml2 {prefix} {prefix} 461COPY --from=openbmc-phosphor-logging {prefix} {prefix} 462RUN curl -L https://github.com/openbmc/phosphor-objmgr/archive/{pkg_rev('openbmc/phosphor-objmgr')}.tar.gz | tar -xz && \ 463cd phosphor-objmgr-* && \ 464./bootstrap.sh && \ 465./configure {configure_flags} && \ 466make -j{proc_count} && \ 467make install 468 469FROM openbmc-base as openbmc-open-power-pdbg 470RUN curl -L https://github.com/open-power/pdbg/archive/{pkg_rev('open-power/pdbg')}.tar.gz | tar -xz && \ 471cd pdbg-* && \ 472./bootstrap.sh && \ 473./configure {configure_flags} && \ 474make -j{proc_count} && \ 475make install 476 477FROM openbmc-base as openbmc-pldm 478COPY --from=openbmc-sdbusplus {prefix} {prefix} 479COPY --from=openbmc-sdeventplus {prefix} {prefix} 480COPY --from=openbmc-boost {prefix} {prefix} 481COPY --from=openbmc-phosphor-dbus-interfaces {prefix} {prefix} 482COPY --from=openbmc-phosphor-logging {prefix} {prefix} 483COPY --from=openbmc-json {prefix} {prefix} 484COPY --from=openbmc-CLI11 {prefix} {prefix} 485RUN curl -L https://github.com/openbmc/pldm/archive/{pkg_rev('openbmc/pldm')}.tar.gz | tar -xz && \ 486cd pldm-* && \ 487meson build {meson_flags} -Dlibpldm-only=enabled -Doem-ibm=enabled -Dtests=disabled && \ 488ninja -C build && \ 489ninja -C build install 490 491# Build the final output image 492FROM openbmc-base 493{copy_cmds} 494 495# Some of our infrastructure still relies on the presence of this file 496# even though it is no longer needed to rebuild the docker environment 497# NOTE: The file is sorted to ensure the ordering is stable. 498RUN echo '{depcache}' > /tmp/depcache 499 500# Final configuration for the workspace 501RUN grep -q {gid} /etc/group || groupadd -g {gid} {username} 502RUN mkdir -p "{os.path.dirname(homedir)}" 503RUN grep -q {uid} /etc/passwd || useradd -d {homedir} -m -u {uid} -g {gid} {username} 504RUN sed -i '1iDefaults umask=000' /etc/sudoers 505RUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers 506 507{proxy_cmd} 508 509RUN /bin/bash 510""" 511 512# Do the docker build. 513for line in docker.build( 514 proxy_args, 515 "--network=host", 516 "-t", 517 docker_image_name, 518 "-", 519 _in=dockerfile, 520 _iter=True, 521): 522 print(line, end="", flush=True) 523