xref: /openbmc/openbmc-build-scripts/scripts/build-unit-test-docker (revision a18d9c57285d69dfd6c10424e763e8f457654344) 
1#!/usr/bin/env python3
2#
3# Build the required docker image to run package unit tests
4#
5# Script Variables:
6#   DOCKER_IMG_NAME:  <optional, the name of the docker image to generate>
7#                     default is openbmc/ubuntu-unit-test
8#   DISTRO:           <optional, the distro to build a docker image against>
9#                     default is ubuntu:eoan
10#   BRANCH:           <optional, branch to build from each of the openbmc/
11#                     repositories>
12#                     default is master, which will be used if input branch not
13#                     provided or not found
14#   UBUNTU_MIRROR:    <optional, the URL of a mirror of Ubuntu to override the
15#                     default ones in /etc/apt/sources.list>
16#                     default is empty, and no mirror is used.
17#   http_proxy        The HTTP address of the proxy server to connect to.
18#                     Default: "", proxy is not setup if this is not set
19
20import os
21import sys
22from datetime import date
23from hashlib import sha256
24from sh import docker, git, nproc, uname
25
26# Read a bunch of environment variables.
27docker_image_name = os.environ.get("DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test")
28distro = os.environ.get("DISTRO", "ubuntu:focal")
29branch = os.environ.get("BRANCH", "master")
30ubuntu_mirror = os.environ.get("UBUNTU_MIRROR")
31http_proxy = os.environ.get("http_proxy")
32prefix = "/usr/local"
33
34# Set up some common variables.
35proc_count = nproc().strip()
36username = os.environ.get("USER")
37homedir = os.environ.get("HOME")
38gid = os.getgid()
39uid = os.getuid()
40
41# Determine the architecture for Docker.
42arch = uname("-m").strip()
43if arch == "ppc64le":
44    docker_base = "ppc64le/"
45elif arch == "x86_64":
46    docker_base = ""
47else:
48    print(f"Unsupported system architecture({arch}) found for docker image")
49    sys.exit(1)
50
51# Packages to include in image.
52packages = {
53    "boost": {
54        "rev": "1.74.0",
55        "url": (
56            lambda pkg, rev: f"https://dl.bintray.com/boostorg/release/{rev}/source/{pkg}_{rev.replace('.', '_')}.tar.bz2"
57        ),
58        "build_type": "custom",
59        "build_steps": [
60            f"./bootstrap.sh --prefix={prefix} --with-libraries=context,coroutine",
61            "./b2",
62            f"./b2 install --prefix={prefix}",
63        ],
64    },
65    "USCiLab/cereal": {
66        "rev": "v1.3.0",
67        "build_type": "custom",
68        "build_steps": [f"cp -a include/cereal/ {prefix}/include/"],
69    },
70    "catchorg/Catch2": {
71        "rev": "v2.12.2",
72        "build_type": "cmake",
73        "config_flags": ["-DBUILD_TESTING=OFF", "-DCATCH_INSTALL_DOCS=OFF"],
74    },
75    "CLIUtils/CLI11": {
76        "rev": "v1.9.0",
77        "build_type": "cmake",
78        "config_flags": [
79            "-DBUILD_TESTING=OFF",
80            "-DCLI11_BUILD_DOCS=OFF",
81            "-DCLI11_BUILD_EXAMPLES=OFF",
82        ],
83    },
84    "fmtlib/fmt": {
85        "rev": "6.2.1",
86        "build_type": "cmake",
87        "config_flags": [
88            "-DFMT_DOC=OFF",
89            "-DFMT_TEST=OFF",
90        ],
91    },
92    # Snapshot from 2020-01-03
93    "Naios/function2": {
94        "rev": "3a0746bf5f601dfed05330aefcb6854354fce07d",
95        "build_type": "custom",
96        "build_steps": [
97            f"mkdir {prefix}/include/function2",
98            f"cp include/function2/function2.hpp {prefix}/include/function2/",
99        ],
100    },
101    # Snapshot from 2020-02-13
102    "google/googletest": {
103        "rev": "23b2a3b1cf803999fb38175f6e9e038a4495c8a5",
104        "build_type": "cmake",
105        "config_env": ["CXXFLAGS=-std=c++17"],
106        "config_flags": ["-DTHREADS_PREFER_PTHREAD_FLAG=ON"],
107    },
108    # Release 2020-08-06
109    "nlohmann/json": {
110        "rev": "v3.9.1",
111        "build_type": "custom",
112        "build_steps": [
113            f"mkdir {prefix}/include/nlohmann",
114            f"cp include/nlohmann/json.hpp {prefix}/include/nlohmann",
115            f"ln -s {prefix}/include/nlohmann/json.hpp {prefix}/include/json.hpp",
116        ],
117    },
118    # Snapshot from 2019-05-24
119    "linux-test-project/lcov": {
120        "rev": "75fbae1cfc5027f818a0bb865bf6f96fab3202da",
121        "build_type": "make",
122    },
123    # dev-5.0 2019-05-03
124    "openbmc/linux": {
125        "rev": "8bf6567e77f7aa68975b7c9c6d044bba690bf327",
126        "build_type": "custom",
127        "build_steps": [
128            f"make -j{proc_count} defconfig",
129            f"make INSTALL_HDR_PATH={prefix} headers_install",
130        ],
131    },
132    # Snapshot from 2019-09-03
133    "LibVNC/libvncserver": {
134        "rev": "1354f7f1bb6962dab209eddb9d6aac1f03408110",
135        "build_type": "cmake",
136    },
137    "martinmoene/span-lite": {
138        "rev": "v0.7.0",
139        "build_type": "cmake",
140        "config_flags": [
141            "-DSPAN_LITE_OPT_BUILD_TESTS=OFF",
142        ],
143    },
144    # version from meta-openembedded/meta-oe/recipes-support/libtinyxml2/libtinyxml2_5.0.1.bb
145    "leethomason/tinyxml2": {
146        "rev": "37bc3aca429f0164adf68c23444540b4a24b5778",
147        "build_type": "cmake",
148    },
149    # version from /meta-openembedded/meta-oe/recipes-devtools/boost-url/boost-url_git.bb
150    "CPPAlliance/url": {
151        "rev": "a56ae0df6d3078319755fbaa67822b4fa7fd352b",
152        "build_type": "cmake",
153        "config_flags": [
154            "-DBOOST_URL_BUILD_EXAMPLES=OFF",
155            "-DBOOST_URL_BUILD_TESTS=OFF",
156            "-DBOOST_URL_STANDALONE=ON",
157        ],
158    },
159    # version from meta-openembedded/meta-oe/recipes-devtools/valijson/valijson_git.bb
160    "tristanpenman/valijson": {
161        "rev": "c2f22fddf599d04dc33fcd7ed257c698a05345d9",
162        "build_type": "cmake",
163        "config_flags": [
164            "-DBUILD_TESTS=0",
165            "-DINSTALL_HEADERS=1",
166        ],
167    },
168    # version from meta-openembedded/meta-oe/recipes-devtools/nlohmann-fifo/nlohmann-fifo_git.bb
169    "nlohmann/fifo_map": {
170        "rev": "0dfbf5dacbb15a32c43f912a7e66a54aae39d0f9",
171        "build_type": "custom",
172        "build_steps": [f"cp src/fifo_map.hpp {prefix}/include/"],
173    },
174    "open-power/pdbg": {"build_type": "autoconf"},
175    "openbmc/gpioplus": {
176        "depends": ["openbmc/stdplus"],
177        "build_type": "meson",
178        "config_flags": [
179            "-Dexamples=false",
180            "-Dtests=disabled",
181        ],
182    },
183    "openbmc/phosphor-dbus-interfaces": {
184        "depends": ["openbmc/sdbusplus"],
185        "build_type": "meson",
186        "config_flags": [
187            "-Ddata_com_ibm=true",
188            "-Ddata_org_open_power=true",
189        ],
190    },
191    "openbmc/phosphor-logging": {
192        "depends": [
193            "USCiLab/cereal",
194            "nlohmann/fifo_map",
195            "openbmc/phosphor-dbus-interfaces",
196            "openbmc/sdbusplus",
197            "openbmc/sdeventplus",
198        ],
199        "build_type": "autoconf",
200        "config_flags": [
201            "--enable-metadata-processing",
202            f"YAML_DIR={prefix}/share/phosphor-dbus-yaml/yaml",
203        ],
204    },
205    "openbmc/phosphor-objmgr": {
206        "depends": [
207            "boost",
208            "leethomason/tinyxml2",
209            "openbmc/phosphor-logging",
210            "openbmc/sdbusplus",
211        ],
212        "build_type": "autoconf",
213    },
214    "openbmc/pldm": {
215        "depends": [
216            "CLIUtils/CLI11",
217            "boost",
218            "nlohmann/json",
219            "openbmc/phosphor-dbus-interfaces",
220            "openbmc/phosphor-logging",
221            "openbmc/sdbusplus",
222            "openbmc/sdeventplus",
223        ],
224        "build_type": "meson",
225        "config_flags": [
226            "-Dlibpldm-only=enabled",
227            "-Doem-ibm=enabled",
228            "-Dtests=disabled",
229        ],
230    },
231    "openbmc/sdbusplus": {
232        "build_type": "meson",
233        "custom_post_dl": [
234            "cd tools",
235            f"./setup.py install --root=/ --prefix={prefix}",
236            "cd ..",
237        ],
238        "config_flags": [
239            "-Dexamples=disabled",
240            "-Dtests=disabled",
241        ],
242    },
243    "openbmc/sdeventplus": {
244        "depends": ["Naios/function2", "openbmc/stdplus"],
245        "build_type": "meson",
246        "config_flags": [
247            "-Dexamples=false",
248            "-Dtests=disabled",
249        ],
250    },
251    "openbmc/stdplus": {
252        "depends": ["fmtlib/fmt", "martinmoene/span-lite"],
253        "build_type": "meson",
254        "config_flags": [
255            "-Dexamples=false",
256            "-Dtests=disabled",
257        ],
258    },
259}
260
261
262def pkg_rev(pkg):
263    return packages[pkg]["rev"]
264
265
266def pkg_stagename(pkg):
267    return pkg.replace("/", "-").lower()
268
269
270def pkg_url(pkg):
271    if "url" in packages[pkg]:
272        return packages[pkg]["url"](pkg, pkg_rev(pkg))
273    return f"https://github.com/{pkg}/archive/{pkg_rev(pkg)}.tar.gz"
274
275
276def pkg_download(pkg):
277    url = pkg_url(pkg)
278    if ".tar." not in url:
279        raise NotImplementedError(f"Unhandled download type for {pkg}: {url}")
280    cmd = f"curl -L {url} | tar -x"
281    if url.endswith(".bz2"):
282        cmd += "j"
283    if url.endswith(".gz"):
284        cmd += "z"
285    return cmd
286
287
288def pkg_copycmds(pkg=None):
289    pkgs = []
290    if pkg:
291        if "depends" not in packages[pkg]:
292            return ""
293        pkgs = sorted(packages[pkg]["depends"])
294    else:
295        pkgs = sorted(packages.keys())
296
297    copy_cmds = ""
298    for p in pkgs:
299        copy_cmds += f"COPY --from={packages[p]['__tag']} {prefix} {prefix}\n"
300        # Workaround for upstream docker bug and multiple COPY cmds
301        # https://github.com/moby/moby/issues/37965
302        copy_cmds += "RUN true\n"
303    return copy_cmds
304
305
306def pkg_cd_srcdir(pkg):
307    return f"cd {pkg.split('/')[-1]}* && "
308
309
310def pkg_build(pkg):
311    result = f"RUN {pkg_download(pkg)} && "
312    result += pkg_cd_srcdir(pkg)
313
314    if "custom_post_dl" in packages[pkg]:
315        result += " && ".join(packages[pkg]["custom_post_dl"]) + " && "
316
317    build_type = packages[pkg]["build_type"]
318    if build_type == "autoconf":
319        result += pkg_build_autoconf(pkg)
320    elif build_type == "cmake":
321        result += pkg_build_cmake(pkg)
322    elif build_type == "custom":
323        result += pkg_build_custom(pkg)
324    elif build_type == "make":
325        result += pkg_build_make(pkg)
326    elif build_type == "meson":
327        result += pkg_build_meson(pkg)
328    else:
329        raise NotImplementedError(
330            f"Unhandled build type for {pkg}: {packages[pkg]['build_type']}"
331        )
332
333    return result
334
335
336def pkg_build_autoconf(pkg):
337    options = " ".join(packages[pkg].get("config_flags", []))
338    env = " ".join(packages[pkg].get("config_env", []))
339    result = "./bootstrap.sh && "
340    result += f"{env} ./configure {configure_flags} {options} && "
341    result += f"make -j{proc_count} && "
342    result += "make install "
343    return result
344
345
346def pkg_build_cmake(pkg):
347    options = " ".join(packages[pkg].get("config_flags", []))
348    env = " ".join(packages[pkg].get("config_env", []))
349    result = "mkdir builddir && cd builddir && "
350    result += f"{env} cmake {cmake_flags} {options} .. && "
351    result += "cmake --build . --target all && "
352    result += "cmake --build . --target install && "
353    result += "cd .. "
354    return result
355
356
357def pkg_build_custom(pkg):
358    return " && ".join(packages[pkg].get("build_steps", []))
359
360
361def pkg_build_make(pkg):
362    result = f"make -j{proc_count} && "
363    result += "make install "
364    return result
365
366
367def pkg_build_meson(pkg):
368    options = " ".join(packages[pkg].get("config_flags", []))
369    env = " ".join(packages[pkg].get("config_env", []))
370    result = f"{env} meson builddir {meson_flags} {options} && "
371    result += "ninja -C builddir && ninja -C builddir install "
372    return result
373
374
375def pkg_generate(pkg):
376    if "__pkg_built" in packages[pkg]:
377        return None
378
379    for deppkg in sorted(packages[pkg].get("depends", [])):
380        pkg_generate(deppkg)
381
382    dockerfile = f"""
383FROM {docker_base_img_name}
384{pkg_copycmds(pkg)}
385{pkg_build(pkg)}
386"""
387
388    tag = docker_img_tagname(pkg_stagename(pkg), dockerfile)
389    packages[pkg]["__tag"] = tag
390    docker_img_build(tag, dockerfile)
391
392    packages[pkg]["__pkg_built"] = True
393
394
395def pkg_generate_packages():
396    for pkg in sorted(packages.keys()):
397        pkg_generate(pkg)
398
399
400def docker_img_tagname(pkgname, dockerfile):
401    result = docker_image_name
402    if pkgname:
403        result += "-" + pkgname
404    result += ":" + date.today().isoformat()
405    result += "-" + sha256(dockerfile.encode()).hexdigest()[0:16]
406    return result
407
408
409def docker_img_build(tag, dockerfile):
410    for line in docker.build(
411        proxy_args,
412        "--network=host",
413        "--force-rm",
414        "-t",
415        tag,
416        "-",
417        _in=dockerfile,
418        _iter=True,
419    ):
420        print(line, end="", flush=True)
421
422
423# Look up the HEAD for missing a static rev.
424pkg_lookups = {}
425for pkg in packages.keys():
426    if "rev" in packages[pkg]:
427        continue
428    pkg_lookups[pkg] = git(
429        "ls-remote", "--heads", f"https://github.com/{pkg}", _bg=True
430    )
431for pkg, result in pkg_lookups.items():
432    for line in result.stdout.decode().split("\n"):
433        if f"refs/heads/{branch}" in line:
434            packages[pkg]["rev"] = line.strip().split()[0]
435        elif "refs/heads/master" in line and p not in packages:
436            packages[pkg]["rev"] = line.strip().split()[0]
437
438# Create the contents of the '/tmp/depcache'.
439# This needs to be sorted for consistency.
440depcache = ""
441for pkg in sorted(packages.keys()):
442    depcache += "%s:%s," % (pkg, pkg_rev(pkg))
443
444# Define common flags used for builds
445configure_flags = " ".join(
446    [
447        f"--prefix={prefix}",
448    ]
449)
450cmake_flags = " ".join(
451    [
452        "-DBUILD_SHARED_LIBS=ON",
453        "-DCMAKE_BUILD_TYPE=RelWithDebInfo",
454        f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}",
455        "-GNinja",
456        "-DCMAKE_MAKE_PROGRAM=ninja",
457    ]
458)
459meson_flags = " ".join(
460    [
461        "--wrap-mode=nodownload",
462        f"-Dprefix={prefix}",
463    ]
464)
465
466# Special flags if setting up a deb mirror.
467mirror = ""
468if "ubuntu" in distro and ubuntu_mirror:
469    mirror = f"""
470RUN echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME) main restricted universe multiverse" > /etc/apt/sources.list && \\
471    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-updates main restricted universe multiverse" >> /etc/apt/sources.list && \\
472    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-security main restricted universe multiverse" >> /etc/apt/sources.list && \\
473    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-proposed main restricted universe multiverse" >> /etc/apt/sources.list && \\
474    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-backports main restricted universe multiverse" >> /etc/apt/sources.list
475"""
476
477# Special flags for proxying.
478proxy_cmd = ""
479proxy_args = []
480if http_proxy:
481    proxy_cmd = f"""
482RUN echo "[http]" >> {homedir}/.gitconfig && \
483    echo "proxy = {http_proxy}" >> {homedir}/.gitconfig
484"""
485    proxy_args.extend(
486        [
487            "--build-arg",
488            f"http_proxy={http_proxy}",
489            "--build-arg",
490            "https_proxy={https_proxy}",
491        ]
492    )
493
494# Create docker image that can run package unit tests
495dockerfile_base = f"""
496FROM {docker_base}{distro}
497
498{mirror}
499
500ENV DEBIAN_FRONTEND noninteractive
501
502ENV PYTHONPATH "/usr/local/lib/python3.8/site-packages/"
503
504# We need the keys to be imported for dbgsym repos
505# New releases have a package, older ones fall back to manual fetching
506# https://wiki.ubuntu.com/Debug%20Symbol%20Packages
507RUN apt-get update && ( apt-get install ubuntu-dbgsym-keyring || ( apt-get install -yy dirmngr && \
508    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622 ) )
509
510# Parse the current repo list into a debug repo list
511RUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list
512
513# Remove non-existent debug repos
514RUN sed -i '/-\(backports\|security\) /d' /etc/apt/sources.list.d/debug.list
515
516RUN cat /etc/apt/sources.list.d/debug.list
517
518RUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \
519    gcc-10 \
520    g++-10 \
521    libc6-dbg \
522    libc6-dev \
523    libtool \
524    bison \
525    libdbus-1-dev \
526    flex \
527    cmake \
528    python3 \
529    python3-dev\
530    python3-yaml \
531    python3-mako \
532    python3-pip \
533    python3-setuptools \
534    python3-git \
535    python3-socks \
536    pkg-config \
537    autoconf \
538    autoconf-archive \
539    libsystemd-dev \
540    systemd \
541    libssl-dev \
542    libevdev-dev \
543    libevdev2-dbgsym \
544    libjpeg-dev \
545    libpng-dev \
546    ninja-build \
547    sudo \
548    curl \
549    git \
550    dbus \
551    iputils-ping \
552    clang-10 \
553    clang-format-10 \
554    clang-tidy-10 \
555    clang-tools-10 \
556    shellcheck \
557    npm \
558    iproute2 \
559    libnl-3-dev \
560    libnl-genl-3-dev \
561    libconfig++-dev \
562    libsnmp-dev \
563    valgrind \
564    valgrind-dbg \
565    libpam0g-dev \
566    xxd \
567    libi2c-dev \
568    wget \
569    libldap2-dev \
570    libprotobuf-dev \
571    libperlio-gzip-perl \
572    libjson-perl \
573    protobuf-compiler \
574    libgpiod-dev \
575    device-tree-compiler \
576    cppcheck \
577    libpciaccess-dev \
578    libmimetic-dev \
579    libxml2-utils \
580    libxml-simple-perl
581
582RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 1000 \
583  --slave /usr/bin/g++ g++ /usr/bin/g++-10 \
584  --slave /usr/bin/gcov gcov /usr/bin/gcov-10 \
585  --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-10 \
586  --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-10
587
588
589RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 1000 \
590  --slave /usr/bin/clang++ clang++ /usr/bin/clang++-10 \
591  --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-10 \
592  --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-10 \
593  --slave /usr/bin/run-clang-tidy.py run-clang-tidy.py /usr/bin/run-clang-tidy-10.py
594
595RUN pip3 install inflection
596RUN pip3 install pycodestyle
597RUN pip3 install jsonschema
598RUN pip3 install meson==0.54.3
599RUN pip3 install protobuf
600"""
601
602# Build the stage docker images.
603docker_base_img_name = docker_img_tagname("base", dockerfile_base)
604docker_img_build(docker_base_img_name, dockerfile_base)
605pkg_generate_packages()
606
607dockerfile = f"""
608# Build the final output image
609FROM {docker_base_img_name}
610{pkg_copycmds()}
611
612# Some of our infrastructure still relies on the presence of this file
613# even though it is no longer needed to rebuild the docker environment
614# NOTE: The file is sorted to ensure the ordering is stable.
615RUN echo '{depcache}' > /tmp/depcache
616
617# Final configuration for the workspace
618RUN grep -q {gid} /etc/group || groupadd -g {gid} {username}
619RUN mkdir -p "{os.path.dirname(homedir)}"
620RUN grep -q {uid} /etc/passwd || useradd -d {homedir} -m -u {uid} -g {gid} {username}
621RUN sed -i '1iDefaults umask=000' /etc/sudoers
622RUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers
623
624{proxy_cmd}
625
626RUN /bin/bash
627"""
628
629# Do the final docker build
630docker_img_build(docker_image_name, dockerfile)
631