1#!/usr/bin/env python3
2#
3# Build the required docker image to run package unit tests
4#
5# Script Variables:
6#   DOCKER_IMG_NAME:  <optional, the name of the docker image to generate>
7#                     default is openbmc/ubuntu-unit-test
8#   DISTRO:           <optional, the distro to build a docker image against>
9#                     default is ubuntu:eoan
10#   BRANCH:           <optional, branch to build from each of the openbmc/
11#                     repositories>
12#                     default is master, which will be used if input branch not
13#                     provided or not found
14#   UBUNTU_MIRROR:    <optional, the URL of a mirror of Ubuntu to override the
15#                     default ones in /etc/apt/sources.list>
16#                     default is empty, and no mirror is used.
17#   http_proxy        The HTTP address of the proxy server to connect to.
18#                     Default: "", proxy is not setup if this is not set
19
20import os
21import sys
22from sh import docker, git, nproc, uname
23
24# Read a bunch of environment variables.
25docker_image_name = os.environ.get("DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test")
26distro = os.environ.get("DISTRO", "ubuntu:focal")
27branch = os.environ.get("BRANCH", "master")
28ubuntu_mirror = os.environ.get("UBUNTU_MIRROR")
29http_proxy = os.environ.get("http_proxy")
30
31# Set up some common variables.
32proc_count = nproc().strip()
33username = os.environ.get("USER")
34homedir = os.environ.get("HOME")
35gid = os.getgid()
36uid = os.getuid()
37
38# Determine the architecture for Docker.
39arch = uname("-m").strip()
40if arch == "ppc64le":
41    docker_base = "ppc64le/"
42elif arch == "x86_64":
43    docker_base = ""
44else:
45    print(f"Unsupported system architecture({arch}) found for docker image")
46    sys.exit(1)
47
48# Packages to include in image.
49packages = {
50    "boost": {
51        "rev": "1.74.0",
52        "url": (
53            lambda pkg, rev: f"https://dl.bintray.com/boostorg/release/{rev}/source/{pkg}_{rev.replace('.', '_')}.tar.bz2"
54        ),
55    },
56    "USCiLab/cereal": {"rev": "v1.3.0"},
57    "catchorg/Catch2": {"rev": "v2.12.2"},
58    "CLIUtils/CLI11": {"rev": "v1.9.0"},
59    "fmtlib/fmt": {"rev": "6.2.1"},
60    # Snapshot from 2020-01-03
61    "Naios/function2": {"rev": "3a0746bf5f601dfed05330aefcb6854354fce07d"},
62    # Snapshot from 2020-02-13
63    "google/googletest": {"rev": "23b2a3b1cf803999fb38175f6e9e038a4495c8a5"},
64    # Release 2020-08-06
65    "nlohmann/json": {"rev": "v3.9.1"},
66    # Snapshot from 2019-05-24
67    "linux-test-project/lcov": {"rev": "75fbae1cfc5027f818a0bb865bf6f96fab3202da"},
68    # dev-5.0 2019-05-03
69    "openbmc/linux": {"rev": "8bf6567e77f7aa68975b7c9c6d044bba690bf327"},
70    # Snapshot from 2019-09-03
71    "LibVNC/libvncserver": {"rev": "1354f7f1bb6962dab209eddb9d6aac1f03408110"},
72    "martinmoene/span-lite": {"rev": "v0.7.0"},
73    # version from meta-openembedded/meta-oe/recipes-support/libtinyxml2/libtinyxml2_5.0.1.bb
74    "leethomason/tinyxml2": {"rev": "37bc3aca429f0164adf68c23444540b4a24b5778"},
75    # version from /meta-openembedded/meta-oe/recipes-devtools/boost-url/boost-url_git.bb
76    "CPPAlliance/url": {"rev": "a56ae0df6d3078319755fbaa67822b4fa7fd352b"},
77    # version from meta-openembedded/meta-oe/recipes-devtools/valijson/valijson_git.bb
78    "tristanpenman/valijson": {"rev": "c2f22fddf599d04dc33fcd7ed257c698a05345d9"},
79    # version from meta-openembedded/meta-oe/recipes-devtools/nlohmann-fifo/nlohmann-fifo_git.bb
80    "nlohmann/fifo_map": {"rev": "0dfbf5dacbb15a32c43f912a7e66a54aae39d0f9"},
81    "open-power/pdbg": {},
82    "openbmc/gpioplus": {},
83    "openbmc/phosphor-dbus-interfaces": {},
84    "openbmc/phosphor-logging": {},
85    "openbmc/phosphor-objmgr": {},
86    "openbmc/pldm": {},
87    "openbmc/sdbusplus": {},
88    "openbmc/sdeventplus": {},
89    "openbmc/stdplus": {},
90}
91
92
93def pkg_rev(pkg):
94    return packages[pkg]["rev"]
95
96
97def pkg_stagename(pkg):
98    if not pkg.startswith("openbmc/"):
99        pkg = "openbmc/" + pkg
100    return pkg.replace("/", "-")
101
102
103def pkg_url(pkg):
104    if "url" in packages[pkg]:
105        return packages[pkg]["url"](pkg, pkg_rev(pkg))
106    return f"https://github.com/{pkg}/archive/{pkg_rev(pkg)}.tar.gz"
107
108
109# Look up the HEAD for missing a static rev.
110pkg_lookups = {}
111for pkg in packages.keys():
112    if "rev" in packages[pkg]:
113        continue
114    pkg_lookups[pkg] = git(
115        "ls-remote", "--heads", f"https://github.com/{pkg}", _bg=True
116    )
117for pkg, result in pkg_lookups.items():
118    for line in result.stdout.decode().split("\n"):
119        if f"refs/heads/{branch}" in line:
120            packages[pkg]["rev"] = line.strip().split()[0]
121        elif "refs/heads/master" in line and p not in packages:
122            packages[pkg]["rev"] = line.strip().split()[0]
123
124# Create the contents of the '/tmp/depcache'.
125# This needs to be sorted for consistency.
126depcache = ""
127for pkg in sorted(packages.keys()):
128    depcache += "%s:%s," % (pkg, pkg_rev(pkg))
129
130# Define common flags used for builds
131prefix = "/usr/local"
132configure_flags = " ".join(
133    [
134        f"--prefix={prefix}",
135    ]
136)
137cmake_flags = " ".join(
138    [
139        "-DCMAKE_BUILD_TYPE=RelWithDebInfo",
140        "-DBUILD_SHARED_LIBS=ON",
141        f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}",
142    ]
143)
144meson_flags = " ".join(
145    [
146        "--wrap-mode=nodownload",
147        f"-Dprefix={prefix}",
148    ]
149)
150
151# Build the commands needed to compose our final image
152# We must sort the packages, otherwise we might produce an unstable
153# docker file and rebuild the image unnecessarily
154copy_cmds = ""
155for pkg in sorted(packages.keys()):
156    copy_cmds += f"COPY --from={pkg_stagename(pkg)} {prefix} {prefix}\n"
157    # Workaround for upstream docker bug and multiple COPY cmds
158    # https://github.com/moby/moby/issues/37965
159    copy_cmds += "RUN true\n"
160
161# Special flags if setting up a deb mirror.
162mirror = ""
163if "ubuntu" in distro and ubuntu_mirror:
164    mirror = f"""
165RUN echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME) main restricted universe multiverse" > /etc/apt/sources.list && \\
166    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-updates main restricted universe multiverse" >> /etc/apt/sources.list && \\
167    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-security main restricted universe multiverse" >> /etc/apt/sources.list && \\
168    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-proposed main restricted universe multiverse" >> /etc/apt/sources.list && \\
169    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-backports main restricted universe multiverse" >> /etc/apt/sources.list
170"""
171
172# Special flags for proxying.
173proxy_cmd = ""
174proxy_args = []
175if http_proxy:
176    proxy_cmd = f"""
177RUN echo "[http]" >> {homedir}/.gitconfig && \
178    echo "proxy = {http_proxy}" >> {homedir}/.gitconfig
179"""
180    proxy_args.extend(
181        [
182            "--build-arg",
183            f"http_proxy={http_proxy}",
184            "--build-arg",
185            "https_proxy={https_proxy}",
186        ]
187    )
188
189# Create docker image that can run package unit tests
190dockerfile = f"""
191FROM {docker_base}{distro} as openbmc-base
192
193{mirror}
194
195ENV DEBIAN_FRONTEND noninteractive
196
197ENV PYTHONPATH "/usr/local/lib/python3.8/site-packages/"
198
199# We need the keys to be imported for dbgsym repos
200# New releases have a package, older ones fall back to manual fetching
201# https://wiki.ubuntu.com/Debug%20Symbol%20Packages
202RUN apt-get update && ( apt-get install ubuntu-dbgsym-keyring || ( apt-get install -yy dirmngr && \
203    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622 ) )
204
205# Parse the current repo list into a debug repo list
206RUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list
207
208# Remove non-existent debug repos
209RUN sed -i '/-\(backports\|security\) /d' /etc/apt/sources.list.d/debug.list
210
211RUN cat /etc/apt/sources.list.d/debug.list
212
213RUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \
214    gcc-10 \
215    g++-10 \
216    libc6-dbg \
217    libc6-dev \
218    libtool \
219    bison \
220    libdbus-1-dev \
221    flex \
222    cmake \
223    python3 \
224    python3-dev\
225    python3-yaml \
226    python3-mako \
227    python3-pip \
228    python3-setuptools \
229    python3-git \
230    python3-socks \
231    pkg-config \
232    autoconf \
233    autoconf-archive \
234    libsystemd-dev \
235    systemd \
236    libssl-dev \
237    libevdev-dev \
238    libevdev2-dbgsym \
239    libjpeg-dev \
240    libpng-dev \
241    ninja-build \
242    sudo \
243    curl \
244    git \
245    dbus \
246    iputils-ping \
247    clang-10 \
248    clang-format-10 \
249    clang-tidy-10 \
250    clang-tools-10 \
251    shellcheck \
252    npm \
253    iproute2 \
254    libnl-3-dev \
255    libnl-genl-3-dev \
256    libconfig++-dev \
257    libsnmp-dev \
258    valgrind \
259    valgrind-dbg \
260    libpam0g-dev \
261    xxd \
262    libi2c-dev \
263    wget \
264    libldap2-dev \
265    libprotobuf-dev \
266    libperlio-gzip-perl \
267    libjson-perl \
268    protobuf-compiler \
269    libgpiod-dev \
270    device-tree-compiler \
271    cppcheck \
272    libpciaccess-dev \
273    libmimetic-dev \
274    libxml2-utils \
275    libxml-simple-perl
276
277RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 1000 \
278  --slave /usr/bin/g++ g++ /usr/bin/g++-10 \
279  --slave /usr/bin/gcov gcov /usr/bin/gcov-10 \
280  --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-10 \
281  --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-10
282
283
284RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 1000 \
285  --slave /usr/bin/clang++ clang++ /usr/bin/clang++-10 \
286  --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-10 \
287  --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-10 \
288  --slave /usr/bin/run-clang-tidy.py run-clang-tidy.py /usr/bin/run-clang-tidy-10.py
289
290RUN pip3 install inflection
291RUN pip3 install pycodestyle
292RUN pip3 install jsonschema
293RUN pip3 install meson==0.54.3
294RUN pip3 install protobuf
295
296FROM openbmc-base as {pkg_stagename('linux-test-project/lcov')}
297RUN curl -L {pkg_url('linux-test-project/lcov')} | tar -xz && \
298cd lcov-* && \
299make -j{proc_count} && \
300make install
301
302FROM openbmc-base as {pkg_stagename('Naios/function2')}
303RUN curl -L {pkg_url('Naios/function2')} | tar -xz && \
304cd function2-* && \
305mkdir {prefix}/include/function2 && \
306cp include/function2/function2.hpp {prefix}/include/function2/
307
308FROM openbmc-base as {pkg_stagename('google/googletest')}
309RUN curl -L {pkg_url('google/googletest')} | tar -xz && \
310cd googletest-* && \
311mkdir build && \
312cd build && \
313CXXFLAGS=-std=c++17 cmake {cmake_flags} -DTHREADS_PREFER_PTHREAD_FLAG=ON .. && \
314make -j{proc_count} && \
315make install
316
317FROM openbmc-base as {pkg_stagename('catchorg/Catch2')}
318RUN curl -L {pkg_url('catchorg/Catch2')} | tar -xz && \
319cd Catch2-* && \
320mkdir build && \
321cd build && \
322cmake {cmake_flags} -DBUILD_TESTING=OFF -DCATCH_INSTALL_DOCS=OFF .. && \
323make -j{proc_count} && \
324make install
325
326FROM openbmc-base as {pkg_stagename('USCiLab/cereal')}
327RUN curl -L {pkg_url('USCiLab/cereal')} | tar -xz && \
328cp -a cereal-*/include/cereal/ {prefix}/include/
329
330FROM openbmc-base as {pkg_stagename('CLIUtils/CLI11')}
331RUN curl -L {pkg_url('CLIUtils/CLI11')} | tar -xz && \
332cd CLI11-* && \
333mkdir build && \
334cd build && \
335cmake {cmake_flags} -DCLI11_BUILD_DOCS=OFF -DBUILD_TESTING=OFF -DCLI11_BUILD_EXAMPLES=OFF .. && \
336make -j{proc_count} && \
337make install
338
339FROM openbmc-base as {pkg_stagename('fmtlib/fmt')}
340RUN curl -L {pkg_url('fmtlib/fmt')} | tar -xz && \
341cd fmt-* && \
342mkdir build && \
343cd build && \
344cmake {cmake_flags} -DFMT_DOC=OFF -DFMT_TEST=OFF .. && \
345make -j{proc_count} && \
346make install
347
348FROM openbmc-base as {pkg_stagename('nlohmann/json')}
349RUN curl -L {pkg_url('nlohmann/json')} | tar -xz && \
350cd json-* && \
351mkdir {prefix}/include/nlohmann && \
352cp include/nlohmann/json.hpp {prefix}/include/nlohmann && \
353ln -s {prefix}/include/nlohmann/json.hpp {prefix}/include/json.hpp
354
355FROM openbmc-base as {pkg_stagename('nlohmann/fifo_map')}
356RUN curl -L {pkg_url('nlohmann/fifo_map')} | tar -xz && \
357cd fifo_map-*/src && cp fifo_map.hpp {prefix}/include/
358
359FROM openbmc-base as {pkg_stagename('martinmoene/span-lite')}
360RUN curl -L {pkg_url('martinmoene/span-lite')} | tar -xz && \
361cd span-lite-* && \
362mkdir build && \
363cd build && \
364cmake {cmake_flags} -DSPAN_LITE_OPT_BUILD_TESTS=OFF .. && \
365make -j{proc_count} && \
366make install
367
368FROM openbmc-base as {pkg_stagename('openbmc/linux')}
369RUN curl -L {pkg_url('openbmc/linux')} | tar -xz && \
370cd linux-* && \
371make -j{proc_count} defconfig && \
372make INSTALL_HDR_PATH=/usr/local headers_install
373
374FROM openbmc-base as {pkg_stagename('boost')}
375RUN curl -L {pkg_url('boost')} | tar -xj && \
376cd boost_*/ && \
377./bootstrap.sh --prefix={prefix} --with-libraries=context,coroutine && \
378./b2 && ./b2 install --prefix={prefix}
379
380FROM openbmc-base as {pkg_stagename('leethomason/tinyxml2')}
381RUN curl -L {pkg_url('leethomason/tinyxml2')} | tar -xz && \
382cd tinyxml2-* && \
383mkdir build && \
384cd build && \
385cmake {cmake_flags} .. && \
386make -j{proc_count} && \
387make install
388
389FROM openbmc-base as {pkg_stagename('CPPAlliance/url')}
390RUN curl -L {pkg_url('CPPAlliance/url')} | tar -xz && \
391cd url-* && \
392mkdir buildir && \
393cd buildir && \
394cmake {cmake_flags} -DBOOST_URL_STANDALONE=ON -DBOOST_URL_BUILD_TESTS=OFF -DBOOST_URL_BUILD_EXAMPLES=OFF .. && \
395make -j{proc_count} && \
396make install
397
398FROM openbmc-base as {pkg_stagename('tristanpenman/valijson')}
399RUN curl -L {pkg_url('tristanpenman/valijson')} | tar -xz && \
400cd valijson-* && \
401mkdir build && \
402cd build && \
403cmake {cmake_flags} -DINSTALL_HEADERS=1 -DBUILD_TESTS=0 .. && \
404make -j{proc_count} && \
405make install
406
407FROM openbmc-base as {pkg_stagename('LibVNC/libvncserver')}
408RUN curl -L {pkg_url('LibVNC/libvncserver')} | tar -xz && \
409cd libvncserver-* && \
410mkdir build && \
411cd build && \
412cmake {cmake_flags} .. && \
413make -j{proc_count} && \
414make install
415
416FROM openbmc-base as {pkg_stagename('openbmc/stdplus')}
417COPY --from={pkg_stagename('fmtlib/fmt')} {prefix} {prefix}
418COPY --from={pkg_stagename('martinmoene/span-lite')} {prefix} {prefix}
419RUN curl -L {pkg_url('openbmc/stdplus')} | tar -xz && \
420cd stdplus-* && \
421meson build {meson_flags} -Dtests=disabled -Dexamples=false && \
422ninja -C build && \
423ninja -C build install
424
425FROM openbmc-base as {pkg_stagename('openbmc/sdbusplus')}
426RUN curl -L {pkg_url('openbmc/sdbusplus')} | tar -xz && \
427cd sdbusplus-* && \
428cd tools && ./setup.py install --root=/ --prefix={prefix} && \
429cd .. && meson build {meson_flags} -Dtests=disabled -Dexamples=disabled && \
430ninja -C build && \
431ninja -C build install
432
433FROM openbmc-base as {pkg_stagename('openbmc/sdeventplus')}
434COPY --from={pkg_stagename('Naios/function2')} {prefix} {prefix}
435COPY --from={pkg_stagename('openbmc/stdplus')} {prefix} {prefix}
436RUN curl -L {pkg_url('openbmc/sdeventplus')} | tar -xz && \
437cd sdeventplus-* && \
438meson build {meson_flags} -Dtests=disabled -Dexamples=false && \
439ninja -C build && \
440ninja -C build install
441
442FROM openbmc-base as {pkg_stagename('openbmc/gpioplus')}
443COPY --from={pkg_stagename('openbmc/stdplus')} {prefix} {prefix}
444RUN curl -L {pkg_url('openbmc/gpioplus')} | tar -xz && \
445cd gpioplus-* && \
446meson build {meson_flags} -Dtests=disabled -Dexamples=false && \
447ninja -C build && \
448ninja -C build install
449
450FROM openbmc-base as {pkg_stagename('openbmc/phosphor-dbus-interfaces')}
451COPY --from={pkg_stagename('openbmc/sdbusplus')} {prefix} {prefix}
452RUN curl -L {pkg_url('openbmc/phosphor-dbus-interfaces')} | tar -xz && \
453cd phosphor-dbus-interfaces-* && \
454meson build {meson_flags} -Ddata_org_open_power=true -Ddata_com_ibm=true && \
455ninja -C build && \
456ninja -C build install
457
458FROM openbmc-base as {pkg_stagename('openbmc/phosphor-logging')}
459COPY --from={pkg_stagename('USCiLab/cereal')} {prefix} {prefix}
460COPY --from={pkg_stagename('openbmc/sdbusplus')} {prefix} {prefix}
461COPY --from={pkg_stagename('openbmc/sdeventplus')} {prefix} {prefix}
462COPY --from={pkg_stagename('openbmc/phosphor-dbus-interfaces')} {prefix} {prefix}
463COPY --from={pkg_stagename('nlohmann/fifo_map')} {prefix} {prefix}
464RUN curl -L {pkg_url('openbmc/phosphor-logging')} | tar -xz && \
465cd phosphor-logging-* && \
466./bootstrap.sh && \
467./configure {configure_flags} --enable-metadata-processing YAML_DIR={prefix}/share/phosphor-dbus-yaml/yaml && \
468make -j{proc_count} && \
469make install
470
471FROM openbmc-base as {pkg_stagename('openbmc/phosphor-objmgr')}
472COPY --from={pkg_stagename('boost')} {prefix} {prefix}
473COPY --from={pkg_stagename('openbmc/sdbusplus')} {prefix} {prefix}
474COPY --from={pkg_stagename('leethomason/tinyxml2')} {prefix} {prefix}
475COPY --from={pkg_stagename('openbmc/phosphor-logging')} {prefix} {prefix}
476RUN curl -L {pkg_url('openbmc/phosphor-objmgr')} | tar -xz && \
477cd phosphor-objmgr-* && \
478./bootstrap.sh && \
479./configure {configure_flags} && \
480make -j{proc_count} && \
481make install
482
483FROM openbmc-base as {pkg_stagename('open-power/pdbg')}
484RUN curl -L {pkg_url('open-power/pdbg')} | tar -xz && \
485cd pdbg-* && \
486./bootstrap.sh && \
487./configure {configure_flags} && \
488make -j{proc_count} && \
489make install
490
491FROM openbmc-base as {pkg_stagename('openbmc/pldm')}
492COPY --from={pkg_stagename('openbmc/sdbusplus')} {prefix} {prefix}
493COPY --from={pkg_stagename('openbmc/sdeventplus')} {prefix} {prefix}
494COPY --from={pkg_stagename('boost')} {prefix} {prefix}
495COPY --from={pkg_stagename('openbmc/phosphor-dbus-interfaces')} {prefix} {prefix}
496COPY --from={pkg_stagename('openbmc/phosphor-logging')} {prefix} {prefix}
497COPY --from={pkg_stagename('nlohmann/json')} {prefix} {prefix}
498COPY --from={pkg_stagename('CLIUtils/CLI11')} {prefix} {prefix}
499RUN curl -L {pkg_url('openbmc/pldm')} | tar -xz && \
500cd pldm-* && \
501meson build {meson_flags} -Dlibpldm-only=enabled -Doem-ibm=enabled -Dtests=disabled && \
502ninja -C build && \
503ninja -C build install
504
505# Build the final output image
506FROM openbmc-base
507{copy_cmds}
508
509# Some of our infrastructure still relies on the presence of this file
510# even though it is no longer needed to rebuild the docker environment
511# NOTE: The file is sorted to ensure the ordering is stable.
512RUN echo '{depcache}' > /tmp/depcache
513
514# Final configuration for the workspace
515RUN grep -q {gid} /etc/group || groupadd -g {gid} {username}
516RUN mkdir -p "{os.path.dirname(homedir)}"
517RUN grep -q {uid} /etc/passwd || useradd -d {homedir} -m -u {uid} -g {gid} {username}
518RUN sed -i '1iDefaults umask=000' /etc/sudoers
519RUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers
520
521{proxy_cmd}
522
523RUN /bin/bash
524"""
525
526# Do the docker build.
527for line in docker.build(
528    proxy_args,
529    "--network=host",
530    "-t",
531    docker_image_name,
532    "-",
533    _in=dockerfile,
534    _iter=True,
535):
536    print(line, end="", flush=True)
537