xref: /openbmc/openbmc-build-scripts/scripts/build-unit-test-docker (revision b16f3e202f7cfd3fc7ed21248f21323b69399288) 
1#!/usr/bin/env python3
2#
3# Build the required docker image to run package unit tests
4#
5# Script Variables:
6#   DOCKER_IMG_NAME:  <optional, the name of the docker image to generate>
7#                     default is openbmc/ubuntu-unit-test
8#   DISTRO:           <optional, the distro to build a docker image against>
9#                     default is ubuntu:eoan
10#   BRANCH:           <optional, branch to build from each of the openbmc/
11#                     repositories>
12#                     default is master, which will be used if input branch not
13#                     provided or not found
14#   UBUNTU_MIRROR:    <optional, the URL of a mirror of Ubuntu to override the
15#                     default ones in /etc/apt/sources.list>
16#                     default is empty, and no mirror is used.
17#   http_proxy        The HTTP address of the proxy server to connect to.
18#                     Default: "", proxy is not setup if this is not set
19
20import os
21import sys
22import threading
23from datetime import date
24from hashlib import sha256
25from sh import docker, git, nproc, uname
26
27# Read a bunch of environment variables.
28docker_image_name = os.environ.get("DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test")
29distro = os.environ.get("DISTRO", "ubuntu:focal")
30branch = os.environ.get("BRANCH", "master")
31ubuntu_mirror = os.environ.get("UBUNTU_MIRROR")
32http_proxy = os.environ.get("http_proxy")
33prefix = "/usr/local"
34
35# Set up some common variables.
36proc_count = nproc().strip()
37username = os.environ.get("USER")
38homedir = os.environ.get("HOME")
39gid = os.getgid()
40uid = os.getuid()
41
42# Determine the architecture for Docker.
43arch = uname("-m").strip()
44if arch == "ppc64le":
45    docker_base = "ppc64le/"
46elif arch == "x86_64":
47    docker_base = ""
48else:
49    print(
50        f"Unsupported system architecture({arch}) found for docker image",
51        file=sys.stderr,
52    )
53    sys.exit(1)
54
55# Packages to include in image.
56packages = {
57    "boost": {
58        "rev": "1.74.0",
59        "url": (
60            lambda pkg, rev: f"https://dl.bintray.com/boostorg/release/{rev}/source/{pkg}_{rev.replace('.', '_')}.tar.bz2"
61        ),
62        "build_type": "custom",
63        "build_steps": [
64            f"./bootstrap.sh --prefix={prefix} --with-libraries=context,coroutine",
65            "./b2",
66            f"./b2 install --prefix={prefix}",
67        ],
68    },
69    "USCiLab/cereal": {
70        "rev": "v1.3.0",
71        "build_type": "custom",
72        "build_steps": [f"cp -a include/cereal/ {prefix}/include/"],
73    },
74    "catchorg/Catch2": {
75        "rev": "v2.12.2",
76        "build_type": "cmake",
77        "config_flags": ["-DBUILD_TESTING=OFF", "-DCATCH_INSTALL_DOCS=OFF"],
78    },
79    "CLIUtils/CLI11": {
80        "rev": "v1.9.0",
81        "build_type": "cmake",
82        "config_flags": [
83            "-DBUILD_TESTING=OFF",
84            "-DCLI11_BUILD_DOCS=OFF",
85            "-DCLI11_BUILD_EXAMPLES=OFF",
86        ],
87    },
88    "fmtlib/fmt": {
89        "rev": "6.2.1",
90        "build_type": "cmake",
91        "config_flags": [
92            "-DFMT_DOC=OFF",
93            "-DFMT_TEST=OFF",
94        ],
95    },
96    # Snapshot from 2020-01-03
97    "Naios/function2": {
98        "rev": "3a0746bf5f601dfed05330aefcb6854354fce07d",
99        "build_type": "custom",
100        "build_steps": [
101            f"mkdir {prefix}/include/function2",
102            f"cp include/function2/function2.hpp {prefix}/include/function2/",
103        ],
104    },
105    # Snapshot from 2020-02-13
106    "google/googletest": {
107        "rev": "23b2a3b1cf803999fb38175f6e9e038a4495c8a5",
108        "build_type": "cmake",
109        "config_env": ["CXXFLAGS=-std=c++17"],
110        "config_flags": ["-DTHREADS_PREFER_PTHREAD_FLAG=ON"],
111    },
112    # Release 2020-08-06
113    "nlohmann/json": {
114        "rev": "v3.9.1",
115        "build_type": "custom",
116        "build_steps": [
117            f"mkdir {prefix}/include/nlohmann",
118            f"cp include/nlohmann/json.hpp {prefix}/include/nlohmann",
119            f"ln -s {prefix}/include/nlohmann/json.hpp {prefix}/include/json.hpp",
120        ],
121    },
122    # Snapshot from 2019-05-24
123    "linux-test-project/lcov": {
124        "rev": "75fbae1cfc5027f818a0bb865bf6f96fab3202da",
125        "build_type": "make",
126    },
127    # dev-5.0 2019-05-03
128    "openbmc/linux": {
129        "rev": "8bf6567e77f7aa68975b7c9c6d044bba690bf327",
130        "build_type": "custom",
131        "build_steps": [
132            f"make -j{proc_count} defconfig",
133            f"make INSTALL_HDR_PATH={prefix} headers_install",
134        ],
135    },
136    # Snapshot from 2019-09-03
137    "LibVNC/libvncserver": {
138        "rev": "1354f7f1bb6962dab209eddb9d6aac1f03408110",
139        "build_type": "cmake",
140    },
141    "martinmoene/span-lite": {
142        "rev": "v0.7.0",
143        "build_type": "cmake",
144        "config_flags": [
145            "-DSPAN_LITE_OPT_BUILD_TESTS=OFF",
146        ],
147    },
148    # version from meta-openembedded/meta-oe/recipes-support/libtinyxml2/libtinyxml2_5.0.1.bb
149    "leethomason/tinyxml2": {
150        "rev": "37bc3aca429f0164adf68c23444540b4a24b5778",
151        "build_type": "cmake",
152    },
153    # version from /meta-openembedded/meta-oe/recipes-devtools/boost-url/boost-url_git.bb
154    "CPPAlliance/url": {
155        "rev": "a56ae0df6d3078319755fbaa67822b4fa7fd352b",
156        "build_type": "cmake",
157        "config_flags": [
158            "-DBOOST_URL_BUILD_EXAMPLES=OFF",
159            "-DBOOST_URL_BUILD_TESTS=OFF",
160            "-DBOOST_URL_STANDALONE=ON",
161        ],
162    },
163    # version from meta-openembedded/meta-oe/recipes-devtools/valijson/valijson_git.bb
164    "tristanpenman/valijson": {
165        "rev": "c2f22fddf599d04dc33fcd7ed257c698a05345d9",
166        "build_type": "cmake",
167        "config_flags": [
168            "-DBUILD_TESTS=0",
169            "-DINSTALL_HEADERS=1",
170        ],
171    },
172    # version from meta-openembedded/meta-oe/recipes-devtools/nlohmann-fifo/nlohmann-fifo_git.bb
173    "nlohmann/fifo_map": {
174        "rev": "0dfbf5dacbb15a32c43f912a7e66a54aae39d0f9",
175        "build_type": "custom",
176        "build_steps": [f"cp src/fifo_map.hpp {prefix}/include/"],
177    },
178    "open-power/pdbg": {"build_type": "autoconf"},
179    "openbmc/gpioplus": {
180        "depends": ["openbmc/stdplus"],
181        "build_type": "meson",
182        "config_flags": [
183            "-Dexamples=false",
184            "-Dtests=disabled",
185        ],
186    },
187    "openbmc/phosphor-dbus-interfaces": {
188        "depends": ["openbmc/sdbusplus"],
189        "build_type": "meson",
190        "config_flags": [
191            "-Ddata_com_ibm=true",
192            "-Ddata_org_open_power=true",
193        ],
194    },
195    "openbmc/phosphor-logging": {
196        "depends": [
197            "USCiLab/cereal",
198            "nlohmann/fifo_map",
199            "openbmc/phosphor-dbus-interfaces",
200            "openbmc/sdbusplus",
201            "openbmc/sdeventplus",
202        ],
203        "build_type": "autoconf",
204        "config_flags": [
205            "--enable-metadata-processing",
206            f"YAML_DIR={prefix}/share/phosphor-dbus-yaml/yaml",
207        ],
208    },
209    "openbmc/phosphor-objmgr": {
210        "depends": [
211            "boost",
212            "leethomason/tinyxml2",
213            "openbmc/phosphor-logging",
214            "openbmc/sdbusplus",
215        ],
216        "build_type": "autoconf",
217    },
218    "openbmc/pldm": {
219        "depends": [
220            "CLIUtils/CLI11",
221            "boost",
222            "nlohmann/json",
223            "openbmc/phosphor-dbus-interfaces",
224            "openbmc/phosphor-logging",
225            "openbmc/sdbusplus",
226            "openbmc/sdeventplus",
227        ],
228        "build_type": "meson",
229        "config_flags": [
230            "-Dlibpldm-only=enabled",
231            "-Doem-ibm=enabled",
232            "-Dtests=disabled",
233        ],
234    },
235    "openbmc/sdbusplus": {
236        "build_type": "meson",
237        "custom_post_dl": [
238            "cd tools",
239            f"./setup.py install --root=/ --prefix={prefix}",
240            "cd ..",
241        ],
242        "config_flags": [
243            "-Dexamples=disabled",
244            "-Dtests=disabled",
245        ],
246    },
247    "openbmc/sdeventplus": {
248        "depends": ["Naios/function2", "openbmc/stdplus"],
249        "build_type": "meson",
250        "config_flags": [
251            "-Dexamples=false",
252            "-Dtests=disabled",
253        ],
254    },
255    "openbmc/stdplus": {
256        "depends": ["fmtlib/fmt", "martinmoene/span-lite"],
257        "build_type": "meson",
258        "config_flags": [
259            "-Dexamples=false",
260            "-Dtests=disabled",
261        ],
262    },
263}
264
265
266def pkg_rev(pkg):
267    return packages[pkg]["rev"]
268
269
270def pkg_stagename(pkg):
271    return pkg.replace("/", "-").lower()
272
273
274def pkg_url(pkg):
275    if "url" in packages[pkg]:
276        return packages[pkg]["url"](pkg, pkg_rev(pkg))
277    return f"https://github.com/{pkg}/archive/{pkg_rev(pkg)}.tar.gz"
278
279
280def pkg_download(pkg):
281    url = pkg_url(pkg)
282    if ".tar." not in url:
283        raise NotImplementedError(f"Unhandled download type for {pkg}: {url}")
284    cmd = f"curl -L {url} | tar -x"
285    if url.endswith(".bz2"):
286        cmd += "j"
287    if url.endswith(".gz"):
288        cmd += "z"
289    return cmd
290
291
292def pkg_copycmds(pkg=None):
293    pkgs = []
294    if pkg:
295        if "depends" not in packages[pkg]:
296            return ""
297        pkgs = sorted(packages[pkg]["depends"])
298    else:
299        pkgs = sorted(packages.keys())
300
301    copy_cmds = ""
302    for p in pkgs:
303        copy_cmds += f"COPY --from={packages[p]['__tag']} {prefix} {prefix}\n"
304        # Workaround for upstream docker bug and multiple COPY cmds
305        # https://github.com/moby/moby/issues/37965
306        copy_cmds += "RUN true\n"
307    return copy_cmds
308
309
310def pkg_cd_srcdir(pkg):
311    return f"cd {pkg.split('/')[-1]}* && "
312
313
314def pkg_build(pkg):
315    result = f"RUN {pkg_download(pkg)} && "
316    result += pkg_cd_srcdir(pkg)
317
318    if "custom_post_dl" in packages[pkg]:
319        result += " && ".join(packages[pkg]["custom_post_dl"]) + " && "
320
321    build_type = packages[pkg]["build_type"]
322    if build_type == "autoconf":
323        result += pkg_build_autoconf(pkg)
324    elif build_type == "cmake":
325        result += pkg_build_cmake(pkg)
326    elif build_type == "custom":
327        result += pkg_build_custom(pkg)
328    elif build_type == "make":
329        result += pkg_build_make(pkg)
330    elif build_type == "meson":
331        result += pkg_build_meson(pkg)
332    else:
333        raise NotImplementedError(
334            f"Unhandled build type for {pkg}: {packages[pkg]['build_type']}"
335        )
336
337    return result
338
339
340def pkg_build_autoconf(pkg):
341    options = " ".join(packages[pkg].get("config_flags", []))
342    env = " ".join(packages[pkg].get("config_env", []))
343    result = "./bootstrap.sh && "
344    result += f"{env} ./configure {configure_flags} {options} && "
345    result += f"make -j{proc_count} && "
346    result += "make install "
347    return result
348
349
350def pkg_build_cmake(pkg):
351    options = " ".join(packages[pkg].get("config_flags", []))
352    env = " ".join(packages[pkg].get("config_env", []))
353    result = "mkdir builddir && cd builddir && "
354    result += f"{env} cmake {cmake_flags} {options} .. && "
355    result += "cmake --build . --target all && "
356    result += "cmake --build . --target install && "
357    result += "cd .. "
358    return result
359
360
361def pkg_build_custom(pkg):
362    return " && ".join(packages[pkg].get("build_steps", []))
363
364
365def pkg_build_make(pkg):
366    result = f"make -j{proc_count} && "
367    result += "make install "
368    return result
369
370
371def pkg_build_meson(pkg):
372    options = " ".join(packages[pkg].get("config_flags", []))
373    env = " ".join(packages[pkg].get("config_env", []))
374    result = f"{env} meson builddir {meson_flags} {options} && "
375    result += "ninja -C builddir && ninja -C builddir install "
376    return result
377
378
379pkg_lock = threading.Lock()
380
381
382def pkg_generate(pkg):
383    class pkg_thread(threading.Thread):
384        def run(self):
385            pkg_lock.acquire()
386            deps = [
387                packages[deppkg]["__thread"]
388                for deppkg in sorted(packages[pkg].get("depends", []))
389            ]
390            pkg_lock.release()
391            for deppkg in deps:
392                deppkg.join()
393
394            dockerfile = f"""
395FROM {docker_base_img_name}
396{pkg_copycmds(pkg)}
397{pkg_build(pkg)}
398"""
399
400            pkg_lock.acquire()
401            tag = docker_img_tagname(pkg_stagename(pkg), dockerfile)
402            packages[pkg]["__tag"] = tag
403            pkg_lock.release()
404
405            try:
406                self.exception = None
407                docker_img_build(pkg, tag, dockerfile)
408            except Exception as e:
409                self.package = pkg
410                self.exception = e
411
412    packages[pkg]["__thread"] = pkg_thread()
413
414
415def pkg_generate_packages():
416    for pkg in packages.keys():
417        pkg_generate(pkg)
418
419    pkg_lock.acquire()
420    pkg_threads = [packages[p]["__thread"] for p in packages.keys()]
421    for t in pkg_threads:
422        t.start()
423    pkg_lock.release()
424
425    for t in pkg_threads:
426        t.join()
427        if t.exception:
428            print(f"Package {t.package} failed!", file=sys.stderr)
429            raise t.exception
430
431
432def docker_img_tagname(pkgname, dockerfile):
433    result = docker_image_name
434    if pkgname:
435        result += "-" + pkgname
436    result += ":" + date.today().isoformat()
437    result += "-" + sha256(dockerfile.encode()).hexdigest()[0:16]
438    return result
439
440
441def docker_img_build(pkg, tag, dockerfile):
442    docker.build(
443        proxy_args,
444        "--network=host",
445        "--force-rm",
446        "-t",
447        tag,
448        "-",
449        _in=dockerfile,
450        _out=(
451            lambda line: print(pkg + ":", line, end="", file=sys.stderr, flush=True)
452        ),
453    )
454
455
456# Look up the HEAD for missing a static rev.
457pkg_lookups = {}
458for pkg in packages.keys():
459    if "rev" in packages[pkg]:
460        continue
461    pkg_lookups[pkg] = git(
462        "ls-remote", "--heads", f"https://github.com/{pkg}", _bg=True
463    )
464for pkg, result in pkg_lookups.items():
465    for line in result.stdout.decode().split("\n"):
466        if f"refs/heads/{branch}" in line:
467            packages[pkg]["rev"] = line.strip().split()[0]
468        elif "refs/heads/master" in line and p not in packages:
469            packages[pkg]["rev"] = line.strip().split()[0]
470
471# Create the contents of the '/tmp/depcache'.
472# This needs to be sorted for consistency.
473depcache = ""
474for pkg in sorted(packages.keys()):
475    depcache += "%s:%s," % (pkg, pkg_rev(pkg))
476
477# Define common flags used for builds
478configure_flags = " ".join(
479    [
480        f"--prefix={prefix}",
481    ]
482)
483cmake_flags = " ".join(
484    [
485        "-DBUILD_SHARED_LIBS=ON",
486        "-DCMAKE_BUILD_TYPE=RelWithDebInfo",
487        f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}",
488        "-GNinja",
489        "-DCMAKE_MAKE_PROGRAM=ninja",
490    ]
491)
492meson_flags = " ".join(
493    [
494        "--wrap-mode=nodownload",
495        f"-Dprefix={prefix}",
496    ]
497)
498
499# Special flags if setting up a deb mirror.
500mirror = ""
501if "ubuntu" in distro and ubuntu_mirror:
502    mirror = f"""
503RUN echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME) main restricted universe multiverse" > /etc/apt/sources.list && \\
504    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-updates main restricted universe multiverse" >> /etc/apt/sources.list && \\
505    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-security main restricted universe multiverse" >> /etc/apt/sources.list && \\
506    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-proposed main restricted universe multiverse" >> /etc/apt/sources.list && \\
507    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-backports main restricted universe multiverse" >> /etc/apt/sources.list
508"""
509
510# Special flags for proxying.
511proxy_cmd = ""
512proxy_args = []
513if http_proxy:
514    proxy_cmd = f"""
515RUN echo "[http]" >> {homedir}/.gitconfig && \
516    echo "proxy = {http_proxy}" >> {homedir}/.gitconfig
517"""
518    proxy_args.extend(
519        [
520            "--build-arg",
521            f"http_proxy={http_proxy}",
522            "--build-arg",
523            "https_proxy={https_proxy}",
524        ]
525    )
526
527# Create docker image that can run package unit tests
528dockerfile_base = f"""
529FROM {docker_base}{distro}
530
531{mirror}
532
533ENV DEBIAN_FRONTEND noninteractive
534
535ENV PYTHONPATH "/usr/local/lib/python3.8/site-packages/"
536
537# We need the keys to be imported for dbgsym repos
538# New releases have a package, older ones fall back to manual fetching
539# https://wiki.ubuntu.com/Debug%20Symbol%20Packages
540RUN apt-get update && ( apt-get install ubuntu-dbgsym-keyring || ( apt-get install -yy dirmngr && \
541    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622 ) )
542
543# Parse the current repo list into a debug repo list
544RUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list
545
546# Remove non-existent debug repos
547RUN sed -i '/-\(backports\|security\) /d' /etc/apt/sources.list.d/debug.list
548
549RUN cat /etc/apt/sources.list.d/debug.list
550
551RUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \
552    gcc-10 \
553    g++-10 \
554    libc6-dbg \
555    libc6-dev \
556    libtool \
557    bison \
558    libdbus-1-dev \
559    flex \
560    cmake \
561    python3 \
562    python3-dev\
563    python3-yaml \
564    python3-mako \
565    python3-pip \
566    python3-setuptools \
567    python3-git \
568    python3-socks \
569    pkg-config \
570    autoconf \
571    autoconf-archive \
572    libsystemd-dev \
573    systemd \
574    libssl-dev \
575    libevdev-dev \
576    libevdev2-dbgsym \
577    libjpeg-dev \
578    libpng-dev \
579    ninja-build \
580    sudo \
581    curl \
582    git \
583    dbus \
584    iputils-ping \
585    clang-10 \
586    clang-format-10 \
587    clang-tidy-10 \
588    clang-tools-10 \
589    shellcheck \
590    npm \
591    iproute2 \
592    libnl-3-dev \
593    libnl-genl-3-dev \
594    libconfig++-dev \
595    libsnmp-dev \
596    valgrind \
597    valgrind-dbg \
598    libpam0g-dev \
599    xxd \
600    libi2c-dev \
601    wget \
602    libldap2-dev \
603    libprotobuf-dev \
604    libperlio-gzip-perl \
605    libjson-perl \
606    protobuf-compiler \
607    libgpiod-dev \
608    device-tree-compiler \
609    cppcheck \
610    libpciaccess-dev \
611    libmimetic-dev \
612    libxml2-utils \
613    libxml-simple-perl
614
615RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 1000 \
616  --slave /usr/bin/g++ g++ /usr/bin/g++-10 \
617  --slave /usr/bin/gcov gcov /usr/bin/gcov-10 \
618  --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-10 \
619  --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-10
620
621
622RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 1000 \
623  --slave /usr/bin/clang++ clang++ /usr/bin/clang++-10 \
624  --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-10 \
625  --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-10 \
626  --slave /usr/bin/run-clang-tidy.py run-clang-tidy.py /usr/bin/run-clang-tidy-10.py
627
628RUN pip3 install inflection
629RUN pip3 install pycodestyle
630RUN pip3 install jsonschema
631RUN pip3 install meson==0.54.3
632RUN pip3 install protobuf
633"""
634
635# Build the stage docker images.
636docker_base_img_name = docker_img_tagname("base", dockerfile_base)
637docker_img_build("base", docker_base_img_name, dockerfile_base)
638pkg_generate_packages()
639
640dockerfile = f"""
641# Build the final output image
642FROM {docker_base_img_name}
643{pkg_copycmds()}
644
645# Some of our infrastructure still relies on the presence of this file
646# even though it is no longer needed to rebuild the docker environment
647# NOTE: The file is sorted to ensure the ordering is stable.
648RUN echo '{depcache}' > /tmp/depcache
649
650# Final configuration for the workspace
651RUN grep -q {gid} /etc/group || groupadd -g {gid} {username}
652RUN mkdir -p "{os.path.dirname(homedir)}"
653RUN grep -q {uid} /etc/passwd || useradd -d {homedir} -m -u {uid} -g {gid} {username}
654RUN sed -i '1iDefaults umask=000' /etc/sudoers
655RUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers
656
657{proxy_cmd}
658
659RUN /bin/bash
660"""
661
662# Do the final docker build
663docker_img_build("final", docker_image_name, dockerfile)
664