1#!/usr/bin/env python3
2#
3# Build the required docker image to run package unit tests
4#
5# Script Variables:
6#   DOCKER_IMG_NAME:  <optional, the name of the docker image to generate>
7#                     default is openbmc/ubuntu-unit-test
8#   DISTRO:           <optional, the distro to build a docker image against>
9#                     default is ubuntu:eoan
10#   BRANCH:           <optional, branch to build from each of the openbmc/
11#                     repositories>
12#                     default is master, which will be used if input branch not
13#                     provided or not found
14#   UBUNTU_MIRROR:    <optional, the URL of a mirror of Ubuntu to override the
15#                     default ones in /etc/apt/sources.list>
16#                     default is empty, and no mirror is used.
17#   http_proxy        The HTTP address of the proxy server to connect to.
18#                     Default: "", proxy is not setup if this is not set
19
20import os
21import sys
22from sh import docker, git, nproc, uname
23
24# Read a bunch of environment variables.
25docker_image_name = os.environ.get("DOCKER_IMAGE_NAME", "openbmc/ubuntu-unit-test")
26distro = os.environ.get("DISTRO", "ubuntu:focal")
27branch = os.environ.get("BRANCH", "master")
28ubuntu_mirror = os.environ.get("UBUNTU_MIRROR")
29http_proxy = os.environ.get("http_proxy")
30prefix = "/usr/local"
31
32# Set up some common variables.
33proc_count = nproc().strip()
34username = os.environ.get("USER")
35homedir = os.environ.get("HOME")
36gid = os.getgid()
37uid = os.getuid()
38
39# Determine the architecture for Docker.
40arch = uname("-m").strip()
41if arch == "ppc64le":
42    docker_base = "ppc64le/"
43elif arch == "x86_64":
44    docker_base = ""
45else:
46    print(f"Unsupported system architecture({arch}) found for docker image")
47    sys.exit(1)
48
49# Packages to include in image.
50packages = {
51    "boost": {
52        "rev": "1.74.0",
53        "url": (
54            lambda pkg, rev: f"https://dl.bintray.com/boostorg/release/{rev}/source/{pkg}_{rev.replace('.', '_')}.tar.bz2"
55        ),
56        "build_type": "custom",
57        "build_steps": [
58            f"./bootstrap.sh --prefix={prefix} --with-libraries=context,coroutine",
59            "./b2",
60            f"./b2 install --prefix={prefix}",
61        ],
62    },
63    "USCiLab/cereal": {
64        "rev": "v1.3.0",
65        "build_type": "custom",
66        "build_steps": [f"cp -a include/cereal/ {prefix}/include/"],
67    },
68    "catchorg/Catch2": {
69        "rev": "v2.12.2",
70        "build_type": "cmake",
71        "config_flags": ["-DBUILD_TESTING=OFF", "-DCATCH_INSTALL_DOCS=OFF"],
72    },
73    "CLIUtils/CLI11": {
74        "rev": "v1.9.0",
75        "build_type": "cmake",
76        "config_flags": [
77            "-DBUILD_TESTING=OFF",
78            "-DCLI11_BUILD_DOCS=OFF",
79            "-DCLI11_BUILD_EXAMPLES=OFF",
80        ],
81    },
82    "fmtlib/fmt": {
83        "rev": "6.2.1",
84        "build_type": "cmake",
85        "config_flags": [
86            "-DFMT_DOC=OFF",
87            "-DFMT_TEST=OFF",
88        ],
89    },
90    # Snapshot from 2020-01-03
91    "Naios/function2": {
92        "rev": "3a0746bf5f601dfed05330aefcb6854354fce07d",
93        "build_type": "custom",
94        "build_steps": [
95            f"mkdir {prefix}/include/function2",
96            f"cp include/function2/function2.hpp {prefix}/include/function2/",
97        ],
98    },
99    # Snapshot from 2020-02-13
100    "google/googletest": {
101        "rev": "23b2a3b1cf803999fb38175f6e9e038a4495c8a5",
102        "build_type": "cmake",
103        "config_env": ["CXXFLAGS=-std=c++17"],
104        "config_flags": ["-DTHREADS_PREFER_PTHREAD_FLAG=ON"],
105    },
106    # Release 2020-08-06
107    "nlohmann/json": {
108        "rev": "v3.9.1",
109        "build_type": "custom",
110        "build_steps": [
111            f"mkdir {prefix}/include/nlohmann",
112            f"cp include/nlohmann/json.hpp {prefix}/include/nlohmann",
113            f"ln -s {prefix}/include/nlohmann/json.hpp {prefix}/include/json.hpp",
114        ],
115    },
116    # Snapshot from 2019-05-24
117    "linux-test-project/lcov": {
118        "rev": "75fbae1cfc5027f818a0bb865bf6f96fab3202da",
119        "build_type": "make",
120    },
121    # dev-5.0 2019-05-03
122    "openbmc/linux": {
123        "rev": "8bf6567e77f7aa68975b7c9c6d044bba690bf327",
124        "build_type": "custom",
125        "build_steps": [
126            f"make -j{proc_count} defconfig",
127            f"make INSTALL_HDR_PATH={prefix} headers_install",
128        ],
129    },
130    # Snapshot from 2019-09-03
131    "LibVNC/libvncserver": {
132        "rev": "1354f7f1bb6962dab209eddb9d6aac1f03408110",
133        "build_type": "cmake",
134    },
135    "martinmoene/span-lite": {
136        "rev": "v0.7.0",
137        "build_type": "cmake",
138        "config_flags": [
139            "-DSPAN_LITE_OPT_BUILD_TESTS=OFF",
140        ],
141    },
142    # version from meta-openembedded/meta-oe/recipes-support/libtinyxml2/libtinyxml2_5.0.1.bb
143    "leethomason/tinyxml2": {
144        "rev": "37bc3aca429f0164adf68c23444540b4a24b5778",
145        "build_type": "cmake",
146    },
147    # version from /meta-openembedded/meta-oe/recipes-devtools/boost-url/boost-url_git.bb
148    "CPPAlliance/url": {
149        "rev": "a56ae0df6d3078319755fbaa67822b4fa7fd352b",
150        "build_type": "cmake",
151        "config_flags": [
152            "-DBOOST_URL_BUILD_EXAMPLES=OFF",
153            "-DBOOST_URL_BUILD_TESTS=OFF",
154            "-DBOOST_URL_STANDALONE=ON",
155        ],
156    },
157    # version from meta-openembedded/meta-oe/recipes-devtools/valijson/valijson_git.bb
158    "tristanpenman/valijson": {
159        "rev": "c2f22fddf599d04dc33fcd7ed257c698a05345d9",
160        "build_type": "cmake",
161        "config_flags": [
162            "-DBUILD_TESTS=0",
163            "-DINSTALL_HEADERS=1",
164        ],
165    },
166    # version from meta-openembedded/meta-oe/recipes-devtools/nlohmann-fifo/nlohmann-fifo_git.bb
167    "nlohmann/fifo_map": {
168        "rev": "0dfbf5dacbb15a32c43f912a7e66a54aae39d0f9",
169        "build_type": "custom",
170        "build_steps": [f"cp src/fifo_map.hpp {prefix}/include/"],
171    },
172    "open-power/pdbg": {"build_type": "autoconf"},
173    "openbmc/gpioplus": {
174        "depends": ["openbmc/stdplus"],
175        "build_type": "meson",
176        "config_flags": [
177            "-Dexamples=false",
178            "-Dtests=disabled",
179        ],
180    },
181    "openbmc/phosphor-dbus-interfaces": {
182        "depends": ["openbmc/sdbusplus"],
183        "build_type": "meson",
184        "config_flags": [
185            "-Ddata_com_ibm=true",
186            "-Ddata_org_open_power=true",
187        ],
188    },
189    "openbmc/phosphor-logging": {
190        "depends": [
191            "USCiLab/cereal",
192            "nlohmann/fifo_map",
193            "openbmc/phosphor-dbus-interfaces",
194            "openbmc/sdbusplus",
195            "openbmc/sdeventplus",
196        ],
197        "build_type": "autoconf",
198        "config_flags": [
199            "--enable-metadata-processing",
200            f"YAML_DIR={prefix}/share/phosphor-dbus-yaml/yaml",
201        ],
202    },
203    "openbmc/phosphor-objmgr": {
204        "depends": [
205            "boost",
206            "leethomason/tinyxml2",
207            "openbmc/phosphor-logging",
208            "openbmc/sdbusplus",
209        ],
210        "build_type": "autoconf",
211    },
212    "openbmc/pldm": {
213        "depends": [
214            "CLIUtils/CLI11",
215            "boost",
216            "nlohmann/json",
217            "openbmc/phosphor-dbus-interfaces",
218            "openbmc/phosphor-logging",
219            "openbmc/sdbusplus",
220            "openbmc/sdeventplus",
221        ],
222        "build_type": "meson",
223        "config_flags": [
224            "-Dlibpldm-only=enabled",
225            "-Doem-ibm=enabled",
226            "-Dtests=disabled",
227        ],
228    },
229    "openbmc/sdbusplus": {
230        "build_type": "meson",
231        "custom_post_dl": [
232            "cd tools",
233            f"./setup.py install --root=/ --prefix={prefix}",
234            "cd ..",
235        ],
236        "config_flags": [
237            "-Dexamples=disabled",
238            "-Dtests=disabled",
239        ],
240    },
241    "openbmc/sdeventplus": {
242        "depends": ["Naios/function2", "openbmc/stdplus"],
243        "build_type": "meson",
244        "config_flags": [
245            "-Dexamples=false",
246            "-Dtests=disabled",
247        ],
248    },
249    "openbmc/stdplus": {
250        "depends": ["fmtlib/fmt", "martinmoene/span-lite"],
251        "build_type": "meson",
252        "config_flags": [
253            "-Dexamples=false",
254            "-Dtests=disabled",
255        ],
256    },
257}
258
259
260def pkg_rev(pkg):
261    return packages[pkg]["rev"]
262
263
264def pkg_stagename(pkg):
265    if not pkg.startswith("openbmc/"):
266        pkg = "openbmc/" + pkg
267    return pkg.replace("/", "-")
268
269
270def pkg_url(pkg):
271    if "url" in packages[pkg]:
272        return packages[pkg]["url"](pkg, pkg_rev(pkg))
273    return f"https://github.com/{pkg}/archive/{pkg_rev(pkg)}.tar.gz"
274
275
276def pkg_download(pkg):
277    url = pkg_url(pkg)
278    if ".tar." not in url:
279        raise NotImplementedError(f"Unhandled download type for {pkg}: {url}")
280    cmd = f"curl -L {url} | tar -x"
281    if url.endswith(".bz2"):
282        cmd += "j"
283    if url.endswith(".gz"):
284        cmd += "z"
285    return cmd
286
287
288def pkg_copycmds(pkg=None):
289    pkgs = []
290    if pkg:
291        if "depends" not in packages[pkg]:
292            return ""
293        pkgs = sorted(packages[pkg]["depends"])
294    else:
295        pkgs = sorted(packages.keys())
296
297    copy_cmds = ""
298    for p in pkgs:
299        copy_cmds += f"COPY --from={pkg_stagename(p)} {prefix} {prefix}\n"
300        # Workaround for upstream docker bug and multiple COPY cmds
301        # https://github.com/moby/moby/issues/37965
302        copy_cmds += "RUN true\n"
303    return copy_cmds
304
305
306def pkg_cd_srcdir(pkg):
307    return f"cd {pkg.split('/')[-1]}* && "
308
309
310def pkg_build(pkg):
311    result = f"RUN {pkg_download(pkg)} && "
312    result += pkg_cd_srcdir(pkg)
313
314    if "custom_post_dl" in packages[pkg]:
315        result += " && ".join(packages[pkg]["custom_post_dl"]) + " && "
316
317    build_type = packages[pkg]["build_type"]
318    if build_type == "autoconf":
319        result += pkg_build_autoconf(pkg)
320    elif build_type == "cmake":
321        result += pkg_build_cmake(pkg)
322    elif build_type == "custom":
323        result += pkg_build_custom(pkg)
324    elif build_type == "make":
325        result += pkg_build_make(pkg)
326    elif build_type == "meson":
327        result += pkg_build_meson(pkg)
328    else:
329        raise NotImplementedError(
330            f"Unhandled build type for {pkg}: {packages[pkg]['build_type']}"
331        )
332
333    return result
334
335
336def pkg_build_autoconf(pkg):
337    options = " ".join(packages[pkg].get("config_flags", []))
338    env = " ".join(packages[pkg].get("config_env", []))
339    result = "./bootstrap.sh && "
340    result += f"{env} ./configure {configure_flags} {options} && "
341    result += f"make -j{proc_count} && "
342    result += "make install "
343    return result
344
345
346def pkg_build_cmake(pkg):
347    options = " ".join(packages[pkg].get("config_flags", []))
348    env = " ".join(packages[pkg].get("config_env", []))
349    result = "mkdir builddir && cd builddir && "
350    result += f"{env} cmake {cmake_flags} {options} .. && "
351    result += f"make -j{proc_count} && "
352    result += f"make install &&"
353    result += "cd .. "
354    return result
355
356
357def pkg_build_custom(pkg):
358    return " && ".join(packages[pkg].get("build_steps", []))
359
360
361def pkg_build_make(pkg):
362    result = f"make -j{proc_count} && "
363    result += "make install "
364    return result
365
366
367def pkg_build_meson(pkg):
368    options = " ".join(packages[pkg].get("config_flags", []))
369    env = " ".join(packages[pkg].get("config_env", []))
370    result = f"{env} meson builddir {meson_flags} {options} && "
371    result += "ninja -C builddir && ninja -C builddir install "
372    return result
373
374
375def pkg_generate(pkg):
376    if "__pkg_built" in packages[pkg]:
377        return None
378
379    result = ""
380
381    for deppkg in sorted(packages[pkg].get("depends", [])):
382        dep_build = pkg_generate(deppkg)
383        if not dep_build:
384            continue
385        result += dep_build + "\n"
386
387    result += f"FROM openbmc-base as {pkg_stagename(pkg)}\n"
388    result += pkg_copycmds(pkg)
389    result += pkg_build(pkg)
390
391    packages[pkg]["__pkg_built"] = True
392    return result
393
394
395def pkg_generate_packages():
396    result = ""
397    for pkg in sorted(packages.keys()):
398        build = pkg_generate(pkg)
399        if not build:
400            continue
401        result += build + "\n"
402    return result
403
404
405# Look up the HEAD for missing a static rev.
406pkg_lookups = {}
407for pkg in packages.keys():
408    if "rev" in packages[pkg]:
409        continue
410    pkg_lookups[pkg] = git(
411        "ls-remote", "--heads", f"https://github.com/{pkg}", _bg=True
412    )
413for pkg, result in pkg_lookups.items():
414    for line in result.stdout.decode().split("\n"):
415        if f"refs/heads/{branch}" in line:
416            packages[pkg]["rev"] = line.strip().split()[0]
417        elif "refs/heads/master" in line and p not in packages:
418            packages[pkg]["rev"] = line.strip().split()[0]
419
420# Create the contents of the '/tmp/depcache'.
421# This needs to be sorted for consistency.
422depcache = ""
423for pkg in sorted(packages.keys()):
424    depcache += "%s:%s," % (pkg, pkg_rev(pkg))
425
426# Define common flags used for builds
427configure_flags = " ".join(
428    [
429        f"--prefix={prefix}",
430    ]
431)
432cmake_flags = " ".join(
433    [
434        "-DCMAKE_BUILD_TYPE=RelWithDebInfo",
435        "-DBUILD_SHARED_LIBS=ON",
436        f"-DCMAKE_INSTALL_PREFIX:PATH={prefix}",
437    ]
438)
439meson_flags = " ".join(
440    [
441        "--wrap-mode=nodownload",
442        f"-Dprefix={prefix}",
443    ]
444)
445
446# Build the commands needed to compose our final image
447# We must sort the packages, otherwise we might produce an unstable
448# docker file and rebuild the image unnecessarily
449copy_cmds = pkg_copycmds()
450
451# Special flags if setting up a deb mirror.
452mirror = ""
453if "ubuntu" in distro and ubuntu_mirror:
454    mirror = f"""
455RUN echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME) main restricted universe multiverse" > /etc/apt/sources.list && \\
456    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-updates main restricted universe multiverse" >> /etc/apt/sources.list && \\
457    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-security main restricted universe multiverse" >> /etc/apt/sources.list && \\
458    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-proposed main restricted universe multiverse" >> /etc/apt/sources.list && \\
459    echo "deb {ubuntu_mirror} $(. /etc/os-release && echo $VERSION_CODENAME)-backports main restricted universe multiverse" >> /etc/apt/sources.list
460"""
461
462# Special flags for proxying.
463proxy_cmd = ""
464proxy_args = []
465if http_proxy:
466    proxy_cmd = f"""
467RUN echo "[http]" >> {homedir}/.gitconfig && \
468    echo "proxy = {http_proxy}" >> {homedir}/.gitconfig
469"""
470    proxy_args.extend(
471        [
472            "--build-arg",
473            f"http_proxy={http_proxy}",
474            "--build-arg",
475            "https_proxy={https_proxy}",
476        ]
477    )
478
479# Create docker image that can run package unit tests
480dockerfile = f"""
481FROM {docker_base}{distro} as openbmc-base
482
483{mirror}
484
485ENV DEBIAN_FRONTEND noninteractive
486
487ENV PYTHONPATH "/usr/local/lib/python3.8/site-packages/"
488
489# We need the keys to be imported for dbgsym repos
490# New releases have a package, older ones fall back to manual fetching
491# https://wiki.ubuntu.com/Debug%20Symbol%20Packages
492RUN apt-get update && ( apt-get install ubuntu-dbgsym-keyring || ( apt-get install -yy dirmngr && \
493    apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F2EDC64DC5AEE1F6B9C621F0C8CAB6595FDFF622 ) )
494
495# Parse the current repo list into a debug repo list
496RUN sed -n '/^deb /s,^deb [^ ]* ,deb http://ddebs.ubuntu.com ,p' /etc/apt/sources.list >/etc/apt/sources.list.d/debug.list
497
498# Remove non-existent debug repos
499RUN sed -i '/-\(backports\|security\) /d' /etc/apt/sources.list.d/debug.list
500
501RUN cat /etc/apt/sources.list.d/debug.list
502
503RUN apt-get update && apt-get dist-upgrade -yy && apt-get install -yy \
504    gcc-10 \
505    g++-10 \
506    libc6-dbg \
507    libc6-dev \
508    libtool \
509    bison \
510    libdbus-1-dev \
511    flex \
512    cmake \
513    python3 \
514    python3-dev\
515    python3-yaml \
516    python3-mako \
517    python3-pip \
518    python3-setuptools \
519    python3-git \
520    python3-socks \
521    pkg-config \
522    autoconf \
523    autoconf-archive \
524    libsystemd-dev \
525    systemd \
526    libssl-dev \
527    libevdev-dev \
528    libevdev2-dbgsym \
529    libjpeg-dev \
530    libpng-dev \
531    ninja-build \
532    sudo \
533    curl \
534    git \
535    dbus \
536    iputils-ping \
537    clang-10 \
538    clang-format-10 \
539    clang-tidy-10 \
540    clang-tools-10 \
541    shellcheck \
542    npm \
543    iproute2 \
544    libnl-3-dev \
545    libnl-genl-3-dev \
546    libconfig++-dev \
547    libsnmp-dev \
548    valgrind \
549    valgrind-dbg \
550    libpam0g-dev \
551    xxd \
552    libi2c-dev \
553    wget \
554    libldap2-dev \
555    libprotobuf-dev \
556    libperlio-gzip-perl \
557    libjson-perl \
558    protobuf-compiler \
559    libgpiod-dev \
560    device-tree-compiler \
561    cppcheck \
562    libpciaccess-dev \
563    libmimetic-dev \
564    libxml2-utils \
565    libxml-simple-perl
566
567RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 1000 \
568  --slave /usr/bin/g++ g++ /usr/bin/g++-10 \
569  --slave /usr/bin/gcov gcov /usr/bin/gcov-10 \
570  --slave /usr/bin/gcov-dump gcov-dump /usr/bin/gcov-dump-10 \
571  --slave /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-tool-10
572
573
574RUN update-alternatives --install /usr/bin/clang clang /usr/bin/clang-10 1000 \
575  --slave /usr/bin/clang++ clang++ /usr/bin/clang++-10 \
576  --slave /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-10 \
577  --slave /usr/bin/clang-format clang-format /usr/bin/clang-format-10 \
578  --slave /usr/bin/run-clang-tidy.py run-clang-tidy.py /usr/bin/run-clang-tidy-10.py
579
580RUN pip3 install inflection
581RUN pip3 install pycodestyle
582RUN pip3 install jsonschema
583RUN pip3 install meson==0.54.3
584RUN pip3 install protobuf
585
586{pkg_generate_packages()}
587
588# Build the final output image
589FROM openbmc-base
590{copy_cmds}
591
592# Some of our infrastructure still relies on the presence of this file
593# even though it is no longer needed to rebuild the docker environment
594# NOTE: The file is sorted to ensure the ordering is stable.
595RUN echo '{depcache}' > /tmp/depcache
596
597# Final configuration for the workspace
598RUN grep -q {gid} /etc/group || groupadd -g {gid} {username}
599RUN mkdir -p "{os.path.dirname(homedir)}"
600RUN grep -q {uid} /etc/passwd || useradd -d {homedir} -m -u {uid} -g {gid} {username}
601RUN sed -i '1iDefaults umask=000' /etc/sudoers
602RUN echo "{username} ALL=(ALL) NOPASSWD: ALL" >>/etc/sudoers
603
604{proxy_cmd}
605
606RUN /bin/bash
607"""
608
609# Do the docker build.
610for line in docker.build(
611    proxy_args,
612    "--network=host",
613    "-t",
614    docker_image_name,
615    "-",
616    _in=dockerfile,
617    _iter=True,
618):
619    print(line, end="", flush=True)
620