1#!/bin/bash
2###############################################################################
3#
4# This build script is for running the OpenBMC builds as Docker containers.
5#
6###############################################################################
7#
8# Script Variables:
9#  build_scripts_dir  The path of the openbmc-build-scripts directory.
10#                     Default: The directory containing this script
11#  http_proxy         The HTTP address of the proxy server to connect to.
12#                     Default: "", proxy is not setup if this is not set
13#  WORKSPACE          Path of the workspace directory where some intermediate
14#                     files and the images will be saved to.
15#                     Default: "~/{RandomNumber}"
16#  num_cpu            Number of cpu's to give bitbake, default is total amount
17#                     in system
18#  UBUNTU_MIRROR      [optional] The URL of a mirror of Ubuntu to override the
19#                     default ones in /etc/apt/sources.list
20#                     default is empty, and no mirror is used.
21#  ENV_LOCAL_CONF     [optional] The environment variables to inject into the
22#                     build, which will be written into local.conf.
23#                     default is empty.
24#  CONTAINER_ONLY     Set to "true" if you only want to build the docker
25#                     container. The bitbake will not occur in this case.
26#
27# Docker Image Build Variables:
28#  BITBAKE_OPTS       Set to "-c populate_sdk" or whatever other BitBake options
29#                     you'd like to pass into the build.
30#                     Default: "", no options set
31#  build_dir          Path where the actual BitBake build occurs inside the
32#                     container, path cannot be located on network storage.
33#                     Default: "$WORKSPACE/build"
34#  distro             The distro used as the base image for the build image:
35#                     fedora|ubuntu
36#                     Default: "ubuntu"
37#  img_name           The name given to the target build's docker image.
38#                     Default: "openbmc/${distro}:${imgtag}-${target}-${ARCH}"
39#  img_tag            The base docker image distro tag:
40#                     ubuntu: latest|16.04|14.04|trusty|xenial
41#                     fedora: 23|24|25
42#                     Default: "latest"
43#  target             The target we aim to build.  Any system supported by
44#                     the openbmc/openbmc `setup` script is an option.
45#                     repotest is a target to specifically run the CI checks
46#                     Default: "qemuarm"
47#  no_tar             Set to true if you do not want the debug tar built
48#                     Default: "false"
49#  nice_priority      Set nice priority for bitbake command.
50#                     Nice:
51#                       Run with an adjusted niceness, which affects process
52#                       scheduling. Nice values range from -20 (most favorable
53#                       to the process) to 19 (least favorable to the process).
54#                     Default: "", nice is not used if nice_priority is not set
55#
56# Deployment Variables:
57#  obmc_dir           Path of the OpenBMC repo directory used as a reference
58#                     for the build inside the container.
59#                     Default: "${WORKSPACE}/openbmc"
60#  ssc_dir            Path of the OpenBMC shared directory that contains the
61#                     downloads dir and the sstate dir.
62#                     Default: "${HOME}"
63#  xtrct_small_copy_dir
64#                     Directory within build_dir that should be copied to
65#                     xtrct_path. The directory and all parents up to, but not
66#                     including, build_dir will be copied. For example, if
67#                     build_dir is set to "/tmp/openbmc" and this is set to
68#                     "build/tmp", the directory at xtrct_path will have the
69#                     following directory structure:
70#                     xtrct_path
71#                      | - build
72#                        | - tmp
73#                          ...
74#                     Can also be set to the empty string to copy the entire
75#                     contents of build_dir to xtrct_path.
76#                     Default: "deploy/images".
77#
78###############################################################################
79# Trace bash processing. Set -e so when a step fails, we fail the build
80set -xeo pipefail
81
82# Script Variables:
83build_scripts_dir=${build_scripts_dir:-"$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"}
84http_proxy=${http_proxy:-}
85WORKSPACE=${WORKSPACE:-${HOME}/${RANDOM}${RANDOM}}
86num_cpu=${num_cpu:-$(nproc)}
87UBUNTU_MIRROR=${UBUNTU_MIRROR:-""}
88ENV_LOCAL_CONF=${ENV_LOCAL_CONF:-""}
89container_only=${CONTAINER_ONLY:-false}
90
91# Docker Image Build Variables:
92build_dir=${build_dir:-${WORKSPACE}/build}
93distro=${distro:-ubuntu}
94img_tag=${img_tag:-latest}
95target=${target:-qemuarm}
96no_tar=${no_tar:-false}
97nice_priority=${nice_priority:-}
98
99# Deployment variables
100obmc_dir=${obmc_dir:-${WORKSPACE}/openbmc}
101ssc_dir=${ssc_dir:-${HOME}}
102xtrct_small_copy_dir=${xtrct_small_copy_dir:-deploy/images}
103xtrct_path="${obmc_dir}/build/tmp"
104xtrct_copy_timeout="300"
105
106bitbake_target="obmc-phosphor-image"
107PROXY=""
108
109MIRROR=""
110if [[ -n "${UBUNTU_MIRROR}" ]]; then
111    MIRROR="RUN echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME) main restricted universe multiverse\" > /etc/apt/sources.list && \
112        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-updates main restricted universe multiverse\" >> /etc/apt/sources.list && \
113        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-security main restricted universe multiverse\" >> /etc/apt/sources.list && \
114        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-proposed main restricted universe multiverse\" >> /etc/apt/sources.list && \
115        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-backports main restricted universe multiverse\" >> /etc/apt/sources.list"
116fi
117
118# Determine the architecture
119ARCH=$(uname -m)
120
121# Determine the prefix of the Dockerfile's base image
122case ${ARCH} in
123    "ppc64le")
124        DOCKER_BASE="ppc64le/"
125        ;;
126    "x86_64")
127        DOCKER_BASE=""
128        ;;
129    "aarch64")
130        DOCKER_BASE="arm64v8/"
131        ;;
132    *)
133        echo "Unsupported system architecture(${ARCH}) found for docker image"
134        exit 1
135esac
136
137# Timestamp for job
138echo "Build started, $(date)"
139
140# If the obmc_dir directory doesn't exist clone it in
141if [ ! -d "${obmc_dir}" ]; then
142    echo "Clone in openbmc master to ${obmc_dir}"
143    git clone https://github.com/openbmc/openbmc "${obmc_dir}"
144fi
145
146if [[ "$target" = repotest ]]; then
147    DOCKER_IMAGE_NAME=$(./scripts/build-unit-test-docker)
148    docker run --cap-add=sys_admin --rm=true \
149        --network host \
150        --privileged=true \
151        -u "$USER" \
152        -w "${obmc_dir}" -v "${obmc_dir}:${obmc_dir}" \
153        -t "${DOCKER_IMAGE_NAME}" \
154        "${obmc_dir}"/meta-phosphor/scripts/run-repotest
155    exit
156fi
157
158# Make and chown the xtrct_path directory to avoid permission errors
159if [ ! -d "${xtrct_path}" ]; then
160    mkdir -p "${xtrct_path}"
161fi
162chown "${UID}:${GROUPS[0]}" "${xtrct_path}"
163
164# Perform overrides for specific machines as required.
165DISTRO=${DISTRO:-}
166
167# Set build target and BitBake command
168MACHINE="${target}"
169BITBAKE_CMD="source ./setup ${MACHINE} ${build_dir}"
170
171# Configure Docker build
172if [[ "${distro}" == fedora ]];then
173
174    if [[ -n "${http_proxy}" ]]; then
175        PROXY="RUN echo \"proxy=${http_proxy}\" >> /etc/dnf/dnf.conf"
176    fi
177
178    Dockerfile=$(cat << EOF
179  FROM ${DOCKER_BASE}${distro}:${img_tag}
180
181  ${PROXY}
182
183  RUN dnf --refresh install -y \
184      bzip2 \
185      chrpath \
186      cpio \
187      diffstat \
188      file \
189      findutils \
190      gcc \
191      gcc-c++ \
192      git \
193      lz4 \
194      make \
195      patch \
196      perl-bignum \
197      perl-Data-Dumper \
198      perl-Thread-Queue \
199      python3-devel \
200      SDL-devel \
201      socat \
202      subversion \
203      tar \
204      texinfo \
205      wget \
206      which \
207      file \
208      hostname \
209      rpcgen \
210      glibc-langpack-en \
211      glibc-locale-source \
212      zstd
213
214  # Set the locale
215  ENV LANG=en_US.utf8
216  RUN localedef -f UTF-8 -i en_US en_US.UTF-8
217
218  RUN grep -q ${GROUPS[0]} /etc/group || groupadd -g ${GROUPS[0]} ${USER}
219  RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS[0]} ${USER}
220
221  USER ${USER}
222  ENV HOME ${HOME}
223EOF
224    )
225
226elif [[ "${distro}" == ubuntu ]]; then
227
228    if [[ -n "${http_proxy}" ]]; then
229        PROXY="RUN echo \"Acquire::http::Proxy \\"\"${http_proxy}/\\"\";\" > /etc/apt/apt.conf.d/000apt-cacher-ng-proxy"
230    fi
231
232    Dockerfile=$(cat << EOF
233  FROM ${DOCKER_BASE}${distro}:${img_tag}
234
235  ${PROXY}
236  ${MIRROR}
237
238  ENV DEBIAN_FRONTEND noninteractive
239
240  RUN apt-get update && apt-get install -yy \
241      build-essential \
242      chrpath \
243      cpio \
244      debianutils \
245      diffstat \
246      file \
247      gawk \
248      git \
249      iputils-ping \
250      libdata-dumper-simple-perl \
251      liblz4-tool \
252      libsdl1.2-dev \
253      libthread-queue-any-perl \
254      locales \
255      python3 \
256      socat \
257      subversion \
258      texinfo \
259      vim \
260      wget \
261      zstd
262
263  # Set the locale
264  RUN locale-gen en_US.UTF-8
265  ENV LANG en_US.UTF-8
266  ENV LANGUAGE en_US:en
267  ENV LC_ALL en_US.UTF-8
268
269  # Latest Ubuntu added a default user (ubuntu), which takes 1000 UID.
270  # If the user calling this build script happens to also have a UID of 1000
271  # then the container no longer will work. Delete the new ubuntu user
272  # so there is no conflict
273  RUN if id ubuntu > /dev/null 2>&1; then userdel -r ubuntu > /dev/null 2>&1; fi
274  RUN grep -q ${GROUPS[0]} /etc/group || groupadd -g ${GROUPS[0]} ${USER}
275  RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS[0]} ${USER}
276
277  USER ${USER}
278  ENV HOME ${HOME}
279EOF
280    )
281fi
282
283# Create the Docker run script
284export PROXY_HOST=${http_proxy/#http*:\/\/}
285export PROXY_HOST=${PROXY_HOST/%:[0-9]*}
286export PROXY_PORT=${http_proxy/#http*:\/\/*:}
287
288mkdir -p "${WORKSPACE}"
289
290# Determine command for bitbake image build
291if [ "$no_tar" = "false" ]; then
292    bitbake_target="${bitbake_target} obmc-phosphor-debug-tarball"
293fi
294
295cat > "${WORKSPACE}"/build.sh << EOF_SCRIPT
296#!/bin/bash
297
298set -xeo pipefail
299
300# Go into the OpenBMC directory, the build will handle changing directories
301cd ${obmc_dir}
302
303# Set up proxies
304export ftp_proxy=${http_proxy}
305export http_proxy=${http_proxy}
306export https_proxy=${http_proxy}
307
308mkdir -p ${WORKSPACE}/bin
309
310# Configure proxies for BitBake
311if [[ -n "${http_proxy}" ]]; then
312
313  cat > ${WORKSPACE}/bin/git-proxy << \EOF_GIT
314  #!/bin/bash
315  # \$1 = hostname, \$2 = port
316  PROXY=${PROXY_HOST}
317  PROXY_PORT=${PROXY_PORT}
318  exec socat STDIO PROXY:\${PROXY}:\${1}:\${2},proxyport=\${PROXY_PORT}
319EOF_GIT
320
321  chmod a+x ${WORKSPACE}/bin/git-proxy
322  export PATH=${WORKSPACE}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH}
323
324  lock=${HOME}/build-setup.lock
325  flock \${lock} git config --global core.gitProxy ${WORKSPACE}/bin/git-proxy
326  flock \${lock} git config --global http.proxy ${http_proxy}
327
328  flock \${lock} mkdir -p ~/.subversion
329  flock \${lock} cat > ~/.subversion/servers << EOF_SVN
330  [global]
331  http-proxy-host = ${PROXY_HOST}
332  http-proxy-port = ${PROXY_PORT}
333EOF_SVN
334
335  flock \${lock} cat > ~/.wgetrc << EOF_WGETRC
336  https_proxy = ${http_proxy}
337  http_proxy = ${http_proxy}
338  use_proxy = on
339EOF_WGETRC
340
341  flock \${lock} cat > ~/.curlrc << EOF_CURLRC
342  proxy = ${PROXY_HOST}:${PROXY_PORT}
343EOF_CURLRC
344fi
345
346# Source our build env
347${BITBAKE_CMD}
348
349if [[ -z "${MACHINE}" ]]; then
350  echo "MACHINE is not configured for ${target}"
351  exit 1
352fi
353
354export MACHINE="${MACHINE}"
355if [[ -z "${DISTRO}" ]]; then
356  echo "DISTRO is not configured for ${target} so will use default"
357  unset DISTRO
358else
359  export DISTRO="${DISTRO}"
360fi
361
362# bitbake requires SDKMACHINE be x86
363export SDKMACHINE=x86_64
364
365# Custom BitBake config settings
366cat >> conf/local.conf << EOF_CONF
367BB_NUMBER_THREADS = "$num_cpu"
368PARALLEL_MAKE = "-j$num_cpu"
369INHERIT += "rm_work"
370BB_GENERATE_MIRROR_TARBALLS = "1"
371DL_DIR="${ssc_dir}/bitbake_downloads"
372SSTATE_DIR="${ssc_dir}/bitbake_sharedstatecache"
373USER_CLASSES += "buildstats"
374INHERIT:remove = "uninative"
375TMPDIR="${build_dir}"
376${ENV_LOCAL_CONF}
377EOF_CONF
378
379# Kick off a build
380if [[ -n "${nice_priority}" ]]; then
381    nice -${nice_priority} bitbake -k ${BITBAKE_OPTS} ${bitbake_target}
382else
383    bitbake -k ${BITBAKE_OPTS} ${bitbake_target}
384fi
385
386# Copy internal build directory into xtrct_path directory
387if [[ ${xtrct_small_copy_dir} ]]; then
388  mkdir -p ${xtrct_path}/${xtrct_small_copy_dir}
389  timeout ${xtrct_copy_timeout} cp -r ${build_dir}/${xtrct_small_copy_dir}/* ${xtrct_path}/${xtrct_small_copy_dir}
390else
391  timeout ${xtrct_copy_timeout} cp -r ${build_dir}/* ${xtrct_path}
392fi
393
394if [[ 0 -ne $? ]]; then
395  echo "Received a non-zero exit code from timeout"
396  exit 1
397fi
398
399EOF_SCRIPT
400
401chmod a+x "${WORKSPACE}/build.sh"
402
403# Give the Docker image a name based on the distro,tag,arch,and target
404img_name=${img_name:-openbmc/${distro}:${img_tag}-${target}-${ARCH}}
405
406# Ensure appropriate docker build output to see progress and identify
407# any issues
408export BUILDKIT_PROGRESS=plain
409
410# Build the Docker image
411docker build --network=host -t "${img_name}" - <<< "${Dockerfile}"
412
413if [[ "$container_only" = "true" ]]; then
414    exit 0
415fi
416
417# If obmc_dir or ssc_dir are ${HOME} or a subdirectory they will not be mounted
418mount_obmc_dir="-v ""${obmc_dir}"":""${obmc_dir}"" "
419mount_ssc_dir="-v ""${ssc_dir}"":""${ssc_dir}"" "
420mount_workspace_dir="-v ""${WORKSPACE}"":""${WORKSPACE}"" "
421if [[ "${obmc_dir}" = "${HOME}/"* || "${obmc_dir}" = "${HOME}" ]];then
422    mount_obmc_dir=""
423fi
424if [[ "${ssc_dir}" = "${HOME}/"* || "${ssc_dir}" = "${HOME}" ]];then
425    mount_ssc_dir=""
426fi
427if [[ "${WORKSPACE}" = "${HOME}/"* || "${WORKSPACE}" = "${HOME}" ]];then
428    mount_workspace_dir=""
429fi
430
431# If we are building on a podman based machine, need to have this set in
432# the env to allow the home mount to work (no impact on non-podman systems)
433export PODMAN_USERNS="keep-id"
434
435# Run the Docker container, execute the build.sh script
436# shellcheck disable=SC2086 # mount commands word-split purposefully
437docker run \
438    --cap-add=sys_admin \
439    --cap-add=sys_nice \
440    --net=host \
441    --rm=true \
442    -e WORKSPACE="${WORKSPACE}" \
443    -w "${HOME}" \
444    -v "${HOME}:${HOME}" \
445    ${mount_obmc_dir} \
446    ${mount_ssc_dir} \
447    ${mount_workspace_dir} \
448    "${img_name}" \
449    "${WORKSPACE}/build.sh"
450
451# To maintain function of resources that used an older path, add a link
452ln -sf "${xtrct_path}/deploy" "${WORKSPACE}/deploy"
453
454# Timestamp for build
455echo "Build completed, $(date)"
456