1#!/bin/bash
2###############################################################################
3#
4# This build script is for running the OpenBMC builds as Docker containers.
5#
6###############################################################################
7#
8# Script Variables:
9#  build_scripts_dir  The path of the openbmc-build-scripts directory.
10#                     Default: The directory containing this script
11#  http_proxy         The HTTP address of the proxy server to connect to.
12#                     Default: "", proxy is not setup if this is not set
13#  WORKSPACE          Path of the workspace directory where some intermediate
14#                     files and the images will be saved to.
15#                     Default: "~/{RandomNumber}"
16#  num_cpu            Number of cpu's to give bitbake, default is total amount
17#                     in system
18#  UBUNTU_MIRROR      [optional] The URL of a mirror of Ubuntu to override the
19#                     default ones in /etc/apt/sources.list
20#                     default is empty, and no mirror is used.
21#  ENV_LOCAL_CONF     [optional] The environment variables to inject into the
22#                     build, which will be written into local.conf.
23#                     default is empty.
24#
25# Docker Image Build Variables:
26#  BITBAKE_OPTS       Set to "-c populate_sdk" or whatever other BitBake options
27#                     you'd like to pass into the build.
28#                     Default: "", no options set
29#  build_dir          Path where the actual BitBake build occurs inside the
30#                     container, path cannot be located on network storage.
31#                     Default: "$WORKSPACE/build"
32#  distro             The distro used as the base image for the build image:
33#                     fedora|ubuntu
34#                     Default: "ubuntu"
35#  img_name           The name given to the target build's docker image.
36#                     Default: "openbmc/${distro}:${imgtag}-${target}-${ARCH}"
37#  img_tag            The base docker image distro tag:
38#                     ubuntu: latest|16.04|14.04|trusty|xenial
39#                     fedora: 23|24|25
40#                     Default: "latest"
41#  target             The target we aim to build.  Any system supported by
42#                     the openbmc/openbmc `setup` script is an option.
43#                     repotest is a target to specifically run the CI checks
44#                     Default: "qemuarm"
45#  no_tar             Set to true if you do not want the debug tar built
46#                     Default: "false"
47#  nice_priority      Set nice priority for bitbake command.
48#                     Nice:
49#                       Run with an adjusted niceness, which affects process
50#                       scheduling. Nice values range from -20 (most favorable
51#                       to the process) to 19 (least favorable to the process).
52#                     Default: "", nice is not used if nice_priority is not set
53#
54# Deployment Variables:
55#  obmc_dir           Path of the OpenBMC repo directory used as a reference
56#                     for the build inside the container.
57#                     Default: "${WORKSPACE}/openbmc"
58#  ssc_dir            Path of the OpenBMC shared directory that contains the
59#                     downloads dir and the sstate dir.
60#                     Default: "${HOME}"
61#  xtrct_small_copy_dir
62#                     Directory within build_dir that should be copied to
63#                     xtrct_path. The directory and all parents up to, but not
64#                     including, build_dir will be copied. For example, if
65#                     build_dir is set to "/tmp/openbmc" and this is set to
66#                     "build/tmp", the directory at xtrct_path will have the
67#                     following directory structure:
68#                     xtrct_path
69#                      | - build
70#                        | - tmp
71#                          ...
72#                     Can also be set to the empty string to copy the entire
73#                     contents of build_dir to xtrct_path.
74#                     Default: "deploy/images".
75#
76###############################################################################
77# Trace bash processing. Set -e so when a step fails, we fail the build
78set -xeo pipefail
79
80# Script Variables:
81build_scripts_dir=${build_scripts_dir:-"$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"}
82http_proxy=${http_proxy:-}
83WORKSPACE=${WORKSPACE:-${HOME}/${RANDOM}${RANDOM}}
84num_cpu=${num_cpu:-$(nproc)}
85UBUNTU_MIRROR=${UBUNTU_MIRROR:-""}
86ENV_LOCAL_CONF=${ENV_LOCAL_CONF:-""}
87
88# Docker Image Build Variables:
89build_dir=${build_dir:-${WORKSPACE}/build}
90distro=${distro:-ubuntu}
91img_tag=${img_tag:-latest}
92target=${target:-qemuarm}
93no_tar=${no_tar:-false}
94nice_priority=${nice_priority:-}
95
96# Deployment variables
97obmc_dir=${obmc_dir:-${WORKSPACE}/openbmc}
98ssc_dir=${ssc_dir:-${HOME}}
99xtrct_small_copy_dir=${xtrct_small_copy_dir:-deploy/images}
100xtrct_path="${obmc_dir}/build/tmp"
101xtrct_copy_timeout="300"
102
103bitbake_target="obmc-phosphor-image"
104PROXY=""
105
106MIRROR=""
107if [[ -n "${UBUNTU_MIRROR}" ]]; then
108    MIRROR="RUN echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME) main restricted universe multiverse\" > /etc/apt/sources.list && \
109        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-updates main restricted universe multiverse\" >> /etc/apt/sources.list && \
110        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-security main restricted universe multiverse\" >> /etc/apt/sources.list && \
111        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-proposed main restricted universe multiverse\" >> /etc/apt/sources.list && \
112        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-backports main restricted universe multiverse\" >> /etc/apt/sources.list"
113fi
114
115# Determine the architecture
116ARCH=$(uname -m)
117
118# Determine the prefix of the Dockerfile's base image
119case ${ARCH} in
120    "ppc64le")
121        DOCKER_BASE="ppc64le/"
122        ;;
123    "x86_64")
124        DOCKER_BASE=""
125        ;;
126    "aarch64")
127        DOCKER_BASE="arm64v8/"
128        ;;
129    *)
130        echo "Unsupported system architecture(${ARCH}) found for docker image"
131        exit 1
132esac
133
134# Timestamp for job
135echo "Build started, $(date)"
136
137# If the obmc_dir directory doesn't exist clone it in
138if [ ! -d "${obmc_dir}" ]; then
139    echo "Clone in openbmc master to ${obmc_dir}"
140    git clone https://github.com/openbmc/openbmc "${obmc_dir}"
141fi
142
143if [[ "$target" = repotest ]]; then
144    DOCKER_IMAGE_NAME=$(./scripts/build-unit-test-docker)
145    docker run --cap-add=sys_admin --rm=true \
146        --network host \
147        --privileged=true \
148        -u "$USER" \
149        -w "${obmc_dir}" -v "${obmc_dir}:${obmc_dir}" \
150        -t "${DOCKER_IMAGE_NAME}" \
151        "${obmc_dir}"/meta-phosphor/scripts/run-repotest
152    exit
153fi
154
155# Make and chown the xtrct_path directory to avoid permission errors
156if [ ! -d "${xtrct_path}" ]; then
157    mkdir -p "${xtrct_path}"
158fi
159chown "${UID}:${GROUPS[0]}" "${xtrct_path}"
160
161# Perform overrides for specific machines as required.
162DISTRO=${DISTRO:-}
163
164# Set build target and BitBake command
165MACHINE="${target}"
166BITBAKE_CMD="source ./setup ${MACHINE} ${build_dir}"
167
168# Configure Docker build
169if [[ "${distro}" == fedora ]];then
170
171    if [[ -n "${http_proxy}" ]]; then
172        PROXY="RUN echo \"proxy=${http_proxy}\" >> /etc/dnf/dnf.conf"
173    fi
174
175    Dockerfile=$(cat << EOF
176  FROM ${DOCKER_BASE}${distro}:${img_tag}
177
178  ${PROXY}
179
180  RUN dnf --refresh install -y \
181      bzip2 \
182      chrpath \
183      cpio \
184      diffstat \
185      file \
186      findutils \
187      gcc \
188      gcc-c++ \
189      git \
190      lz4 \
191      make \
192      patch \
193      perl-bignum \
194      perl-Data-Dumper \
195      perl-Thread-Queue \
196      python3-devel \
197      SDL-devel \
198      socat \
199      subversion \
200      tar \
201      texinfo \
202      wget \
203      which \
204      file \
205      hostname \
206      rpcgen \
207      glibc-langpack-en \
208      glibc-locale-source \
209      zstd
210
211  # Set the locale
212  ENV LANG=en_US.utf8
213  RUN localedef -f UTF-8 -i en_US en_US.UTF-8
214
215  RUN grep -q ${GROUPS[0]} /etc/group || groupadd -g ${GROUPS[0]} ${USER}
216  RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS[0]} ${USER}
217
218  USER ${USER}
219  ENV HOME ${HOME}
220EOF
221    )
222
223elif [[ "${distro}" == ubuntu ]]; then
224
225    if [[ -n "${http_proxy}" ]]; then
226        PROXY="RUN echo \"Acquire::http::Proxy \\"\"${http_proxy}/\\"\";\" > /etc/apt/apt.conf.d/000apt-cacher-ng-proxy"
227    fi
228
229    Dockerfile=$(cat << EOF
230  FROM ${DOCKER_BASE}${distro}:${img_tag}
231
232  ${PROXY}
233  ${MIRROR}
234
235  ENV DEBIAN_FRONTEND noninteractive
236
237  RUN apt-get update && apt-get install -yy \
238      build-essential \
239      chrpath \
240      cpio \
241      debianutils \
242      diffstat \
243      file \
244      gawk \
245      git \
246      iputils-ping \
247      libdata-dumper-simple-perl \
248      liblz4-tool \
249      libsdl1.2-dev \
250      libthread-queue-any-perl \
251      locales \
252      python3 \
253      socat \
254      subversion \
255      texinfo \
256      vim \
257      wget \
258      zstd
259
260  # Set the locale
261  RUN locale-gen en_US.UTF-8
262  ENV LANG en_US.UTF-8
263  ENV LANGUAGE en_US:en
264  ENV LC_ALL en_US.UTF-8
265
266  # Latest Ubuntu added a default user (ubuntu), which takes 1000 UID.
267  # If the user calling this build script happens to also have a UID of 1000
268  # then the container no longer will work. Delete the new ubuntu user
269  # so there is no conflict
270  RUN if id ubuntu > /dev/null 2>&1; then userdel -r ubuntu > /dev/null 2>&1; fi
271  RUN grep -q ${GROUPS[0]} /etc/group || groupadd -g ${GROUPS[0]} ${USER}
272  RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS[0]} ${USER}
273
274  USER ${USER}
275  ENV HOME ${HOME}
276EOF
277    )
278fi
279
280# Create the Docker run script
281export PROXY_HOST=${http_proxy/#http*:\/\/}
282export PROXY_HOST=${PROXY_HOST/%:[0-9]*}
283export PROXY_PORT=${http_proxy/#http*:\/\/*:}
284
285mkdir -p "${WORKSPACE}"
286
287# Determine command for bitbake image build
288if [ "$no_tar" = "false" ]; then
289    bitbake_target="${bitbake_target} obmc-phosphor-debug-tarball"
290fi
291
292cat > "${WORKSPACE}"/build.sh << EOF_SCRIPT
293#!/bin/bash
294
295set -xeo pipefail
296
297# Go into the OpenBMC directory, the build will handle changing directories
298cd ${obmc_dir}
299
300# Set up proxies
301export ftp_proxy=${http_proxy}
302export http_proxy=${http_proxy}
303export https_proxy=${http_proxy}
304
305mkdir -p ${WORKSPACE}/bin
306
307# Configure proxies for BitBake
308if [[ -n "${http_proxy}" ]]; then
309
310  cat > ${WORKSPACE}/bin/git-proxy << \EOF_GIT
311  #!/bin/bash
312  # \$1 = hostname, \$2 = port
313  PROXY=${PROXY_HOST}
314  PROXY_PORT=${PROXY_PORT}
315  exec socat STDIO PROXY:\${PROXY}:\${1}:\${2},proxyport=\${PROXY_PORT}
316EOF_GIT
317
318  chmod a+x ${WORKSPACE}/bin/git-proxy
319  export PATH=${WORKSPACE}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH}
320
321  lock=${HOME}/build-setup.lock
322  flock \${lock} git config --global core.gitProxy ${WORKSPACE}/bin/git-proxy
323  flock \${lock} git config --global http.proxy ${http_proxy}
324
325  flock \${lock} mkdir -p ~/.subversion
326  flock \${lock} cat > ~/.subversion/servers << EOF_SVN
327  [global]
328  http-proxy-host = ${PROXY_HOST}
329  http-proxy-port = ${PROXY_PORT}
330EOF_SVN
331
332  flock \${lock} cat > ~/.wgetrc << EOF_WGETRC
333  https_proxy = ${http_proxy}
334  http_proxy = ${http_proxy}
335  use_proxy = on
336EOF_WGETRC
337
338  flock \${lock} cat > ~/.curlrc << EOF_CURLRC
339  proxy = ${PROXY_HOST}:${PROXY_PORT}
340EOF_CURLRC
341fi
342
343# Source our build env
344${BITBAKE_CMD}
345
346if [[ -z "${MACHINE}" ]]; then
347  echo "MACHINE is not configured for ${target}"
348  exit 1
349fi
350
351export MACHINE="${MACHINE}"
352if [[ -z "${DISTRO}" ]]; then
353  echo "DISTRO is not configured for ${target} so will use default"
354  unset DISTRO
355else
356  export DISTRO="${DISTRO}"
357fi
358
359# bitbake requires SDKMACHINE be x86
360export SDKMACHINE=x86_64
361
362# Custom BitBake config settings
363cat >> conf/local.conf << EOF_CONF
364BB_NUMBER_THREADS = "$num_cpu"
365PARALLEL_MAKE = "-j$num_cpu"
366INHERIT += "rm_work"
367BB_GENERATE_MIRROR_TARBALLS = "1"
368DL_DIR="${ssc_dir}/bitbake_downloads"
369SSTATE_DIR="${ssc_dir}/bitbake_sharedstatecache"
370USER_CLASSES += "buildstats"
371INHERIT:remove = "uninative"
372TMPDIR="${build_dir}"
373${ENV_LOCAL_CONF}
374EOF_CONF
375
376# Kick off a build
377if [[ -n "${nice_priority}" ]]; then
378    nice -${nice_priority} bitbake -k ${BITBAKE_OPTS} ${bitbake_target}
379else
380    bitbake -k ${BITBAKE_OPTS} ${bitbake_target}
381fi
382
383# Copy internal build directory into xtrct_path directory
384if [[ ${xtrct_small_copy_dir} ]]; then
385  mkdir -p ${xtrct_path}/${xtrct_small_copy_dir}
386  timeout ${xtrct_copy_timeout} cp -r ${build_dir}/${xtrct_small_copy_dir}/* ${xtrct_path}/${xtrct_small_copy_dir}
387else
388  timeout ${xtrct_copy_timeout} cp -r ${build_dir}/* ${xtrct_path}
389fi
390
391if [[ 0 -ne $? ]]; then
392  echo "Received a non-zero exit code from timeout"
393  exit 1
394fi
395
396EOF_SCRIPT
397
398chmod a+x "${WORKSPACE}/build.sh"
399
400# Give the Docker image a name based on the distro,tag,arch,and target
401img_name=${img_name:-openbmc/${distro}:${img_tag}-${target}-${ARCH}}
402
403# Ensure appropriate docker build output to see progress and identify
404# any issues
405export BUILDKIT_PROGRESS=plain
406
407# Build the Docker image
408docker build --network=host -t "${img_name}" - <<< "${Dockerfile}"
409
410# If obmc_dir or ssc_dir are ${HOME} or a subdirectory they will not be mounted
411mount_obmc_dir="-v ""${obmc_dir}"":""${obmc_dir}"" "
412mount_ssc_dir="-v ""${ssc_dir}"":""${ssc_dir}"" "
413mount_workspace_dir="-v ""${WORKSPACE}"":""${WORKSPACE}"" "
414if [[ "${obmc_dir}" = "${HOME}/"* || "${obmc_dir}" = "${HOME}" ]];then
415    mount_obmc_dir=""
416fi
417if [[ "${ssc_dir}" = "${HOME}/"* || "${ssc_dir}" = "${HOME}" ]];then
418    mount_ssc_dir=""
419fi
420if [[ "${WORKSPACE}" = "${HOME}/"* || "${WORKSPACE}" = "${HOME}" ]];then
421    mount_workspace_dir=""
422fi
423
424# If we are building on a podman based machine, need to have this set in
425# the env to allow the home mount to work (no impact on non-podman systems)
426export PODMAN_USERNS="keep-id"
427
428# Run the Docker container, execute the build.sh script
429# shellcheck disable=SC2086 # mount commands word-split purposefully
430docker run \
431    --cap-add=sys_admin \
432    --cap-add=sys_nice \
433    --net=host \
434    --rm=true \
435    -e WORKSPACE="${WORKSPACE}" \
436    -w "${HOME}" \
437    -v "${HOME}:${HOME}" \
438    ${mount_obmc_dir} \
439    ${mount_ssc_dir} \
440    ${mount_workspace_dir} \
441    "${img_name}" \
442    "${WORKSPACE}/build.sh"
443
444# To maintain function of resources that used an older path, add a link
445ln -sf "${xtrct_path}/deploy" "${WORKSPACE}/deploy"
446
447# Timestamp for build
448echo "Build completed, $(date)"
449