1#!/bin/bash
2###############################################################################
3#
4# This build script is for running the OpenBMC builds as Docker containers.
5#
6###############################################################################
7#
8# Script Variables:
9#  build_scripts_dir  The path of the openbmc-build-scripts directory.
10#                     Default: The directory containing this script
11#  http_proxy         The HTTP address of the proxy server to connect to.
12#                     Default: "", proxy is not setup if this is not set
13#  WORKSPACE          Path of the workspace directory where some intermediate
14#                     files and the images will be saved to.
15#                     Default: "~/{RandomNumber}"
16#  num_cpu            Number of cpu's to give bitbake, default is total amount
17#                     in system
18#  UBUNTU_MIRROR      [optional] The URL of a mirror of Ubuntu to override the
19#                     default ones in /etc/apt/sources.list
20#                     default is empty, and no mirror is used.
21#  ENV_LOCAL_CONF     [optional] The environment variables to inject into the
22#                     build, which will be written into local.conf.
23#                     default is empty.
24#  CONTAINER_ONLY     Set to "true" if you only want to build the docker
25#                     container. The bitbake will not occur in this case.
26#  DOCKER_REG:        <optional, the URL of a docker registry to utilize
27#                     instead of our default (public.ecr.aws/ubuntu)
28#                     (ex. docker.io or public.ecr.aws/docker/library)
29#
30# Docker Image Build Variables:
31#  BITBAKE_OPTS       Set to "-c populate_sdk" or whatever other BitBake options
32#                     you'd like to pass into the build.
33#                     Default: "", no options set
34#  build_dir          Path where the actual BitBake build occurs inside the
35#                     container, path cannot be located on network storage.
36#                     Default: "$WORKSPACE/build"
37#  distro             The distro used as the base image for the build image:
38#                     fedora|ubuntu. Note that if you chose fedora, you will
39#                     need to also update DOCKER_REG to a supported fedora reg.
40#                     Default: "ubuntu"
41#  img_name           The name given to the target build's docker image.
42#                     Default: "openbmc/${distro}:${imgtag}-${target}-${ARCH}"
43#  img_tag            The base docker image distro tag:
44#                     ubuntu: latest|16.04|14.04|trusty|xenial
45#                     fedora: 23|24|25
46#                     Default: "latest"
47#  target             The target we aim to build.  Any system supported by
48#                     the openbmc/openbmc `setup` script is an option.
49#                     repotest is a target to specifically run the CI checks
50#                     Default: "qemuarm"
51#  no_tar             Set to true if you do not want the debug tar built
52#                     Default: "false"
53#  nice_priority      Set nice priority for bitbake command.
54#                     Nice:
55#                       Run with an adjusted niceness, which affects process
56#                       scheduling. Nice values range from -20 (most favorable
57#                       to the process) to 19 (least favorable to the process).
58#                     Default: "", nice is not used if nice_priority is not set
59#
60# Deployment Variables:
61#  obmc_dir           Path of the OpenBMC repo directory used as a reference
62#                     for the build inside the container.
63#                     Default: "${WORKSPACE}/openbmc"
64#  ssc_dir            Path of the OpenBMC shared directory that contains the
65#                     downloads dir and the sstate dir.
66#                     Default: "${HOME}"
67#  xtrct_small_copy_dir
68#                     Directory within build_dir that should be copied to
69#                     xtrct_path. The directory and all parents up to, but not
70#                     including, build_dir will be copied. For example, if
71#                     build_dir is set to "/tmp/openbmc" and this is set to
72#                     "build/tmp", the directory at xtrct_path will have the
73#                     following directory structure:
74#                     xtrct_path
75#                      | - build
76#                        | - tmp
77#                          ...
78#                     Can also be set to the empty string to copy the entire
79#                     contents of build_dir to xtrct_path.
80#                     Default: "deploy/images".
81#
82###############################################################################
83# Trace bash processing. Set -e so when a step fails, we fail the build
84set -xeo pipefail
85
86# Script Variables:
87build_scripts_dir=${build_scripts_dir:-"$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"}
88http_proxy=${http_proxy:-}
89WORKSPACE=${WORKSPACE:-${HOME}/${RANDOM}${RANDOM}}
90num_cpu=${num_cpu:-$(nproc)}
91UBUNTU_MIRROR=${UBUNTU_MIRROR:-""}
92ENV_LOCAL_CONF=${ENV_LOCAL_CONF:-""}
93container_only=${CONTAINER_ONLY:-false}
94docker_reg=${DOCKER_REG:-"public.ecr.aws/ubuntu"}
95
96# Docker Image Build Variables:
97build_dir=${build_dir:-${WORKSPACE}/build}
98distro=${distro:-ubuntu}
99img_tag=${img_tag:-latest}
100target=${target:-qemuarm}
101no_tar=${no_tar:-false}
102nice_priority=${nice_priority:-}
103
104# Deployment variables
105obmc_dir=${obmc_dir:-${WORKSPACE}/openbmc}
106ssc_dir=${ssc_dir:-${HOME}}
107xtrct_small_copy_dir=${xtrct_small_copy_dir:-deploy/images}
108xtrct_path="${obmc_dir}/build/tmp"
109xtrct_copy_timeout="300"
110
111bitbake_target="obmc-phosphor-image"
112PROXY=""
113
114MIRROR=""
115if [[ -n "${UBUNTU_MIRROR}" ]]; then
116    MIRROR="RUN echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME) main restricted universe multiverse\" > /etc/apt/sources.list && \
117        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-updates main restricted universe multiverse\" >> /etc/apt/sources.list && \
118        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-security main restricted universe multiverse\" >> /etc/apt/sources.list && \
119        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-proposed main restricted universe multiverse\" >> /etc/apt/sources.list && \
120        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-backports main restricted universe multiverse\" >> /etc/apt/sources.list"
121fi
122
123# Timestamp for job
124echo "Build started, $(date)"
125
126# If the obmc_dir directory doesn't exist clone it in
127if [ ! -d "${obmc_dir}" ]; then
128    echo "Clone in openbmc master to ${obmc_dir}"
129    git clone https://github.com/openbmc/openbmc "${obmc_dir}"
130fi
131
132if [[ "$target" = repotest ]]; then
133    DOCKER_IMAGE_NAME=$(./scripts/build-unit-test-docker)
134    docker run --cap-add=sys_admin --rm=true \
135        --network host \
136        --privileged=true \
137        -u "$USER" \
138        -w "${obmc_dir}" -v "${obmc_dir}:${obmc_dir}" \
139        -t "${DOCKER_IMAGE_NAME}" \
140        "${obmc_dir}"/meta-phosphor/scripts/run-repotest
141    exit
142fi
143
144# Make and chown the xtrct_path directory to avoid permission errors
145if [ ! -d "${xtrct_path}" ]; then
146    mkdir -p "${xtrct_path}"
147fi
148chown "${UID}:${GROUPS[0]}" "${xtrct_path}"
149
150# Perform overrides for specific machines as required.
151DISTRO=${DISTRO:-}
152
153# Set build target and BitBake command
154MACHINE="${target}"
155BITBAKE_CMD="source ./setup ${MACHINE} ${build_dir}"
156
157# Configure Docker build
158if [[ "${distro}" == fedora ]];then
159
160    if [[ -n "${http_proxy}" ]]; then
161        PROXY="RUN echo \"proxy=${http_proxy}\" >> /etc/dnf/dnf.conf"
162    fi
163
164    Dockerfile=$(cat << EOF
165  FROM ${docker_reg}/${distro}:${img_tag}
166
167  ${PROXY}
168
169  RUN dnf --refresh install -y \
170      bzip2 \
171      chrpath \
172      cpio \
173      diffstat \
174      file \
175      findutils \
176      gcc \
177      gcc-c++ \
178      git \
179      lz4 \
180      make \
181      patch \
182      perl-bignum \
183      perl-Data-Dumper \
184      perl-Thread-Queue \
185      python3-devel \
186      SDL-devel \
187      socat \
188      subversion \
189      tar \
190      texinfo \
191      wget \
192      which \
193      file \
194      hostname \
195      rpcgen \
196      glibc-langpack-en \
197      glibc-locale-source \
198      zstd
199
200  # Set the locale
201  ENV LANG=en_US.utf8
202  RUN localedef -f UTF-8 -i en_US en_US.UTF-8
203
204  RUN grep -q ${GROUPS[0]} /etc/group || groupadd -g ${GROUPS[0]} ${USER}
205  RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS[0]} ${USER}
206
207  USER ${USER}
208  ENV HOME ${HOME}
209EOF
210    )
211
212elif [[ "${distro}" == ubuntu ]]; then
213
214    if [[ -n "${http_proxy}" ]]; then
215        PROXY="RUN echo \"Acquire::http::Proxy \\"\"${http_proxy}/\\"\";\" > /etc/apt/apt.conf.d/000apt-cacher-ng-proxy"
216    fi
217
218    Dockerfile=$(cat << EOF
219  FROM ${docker_reg}/${distro}:${img_tag}
220
221  ${PROXY}
222  ${MIRROR}
223
224  ENV DEBIAN_FRONTEND noninteractive
225
226  RUN apt-get update && apt-get install -yy \
227      build-essential \
228      chrpath \
229      cpio \
230      debianutils \
231      diffstat \
232      file \
233      gawk \
234      git \
235      iputils-ping \
236      libdata-dumper-simple-perl \
237      lz4 \
238      libsdl1.2-dev \
239      libthread-queue-any-perl \
240      locales \
241      python3 \
242      socat \
243      subversion \
244      texinfo \
245      vim \
246      wget \
247      zstd
248
249  # Set the locale
250  RUN locale-gen en_US.UTF-8
251  ENV LANG en_US.UTF-8
252  ENV LANGUAGE en_US:en
253  ENV LC_ALL en_US.UTF-8
254
255  # Latest Ubuntu added a default user (ubuntu), which takes 1000 UID.
256  # If the user calling this build script happens to also have a UID of 1000
257  # then the container no longer will work. Delete the new ubuntu user
258  # so there is no conflict
259  RUN if id ubuntu > /dev/null 2>&1; then userdel -r ubuntu > /dev/null 2>&1; fi
260  RUN grep -q ${GROUPS[0]} /etc/group || groupadd -g ${GROUPS[0]} ${USER}
261  RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS[0]} ${USER}
262
263  USER ${USER}
264  ENV HOME ${HOME}
265EOF
266    )
267fi
268
269# Create the Docker run script
270export PROXY_HOST=${http_proxy/#http*:\/\/}
271export PROXY_HOST=${PROXY_HOST/%:[0-9]*}
272export PROXY_PORT=${http_proxy/#http*:\/\/*:}
273
274mkdir -p "${WORKSPACE}"
275
276# Determine command for bitbake image build
277if [ "$no_tar" = "false" ]; then
278    bitbake_target="${bitbake_target} obmc-phosphor-debug-tarball"
279fi
280
281cat > "${WORKSPACE}"/build.sh << EOF_SCRIPT
282#!/bin/bash
283
284set -xeo pipefail
285
286# Go into the OpenBMC directory, the build will handle changing directories
287cd ${obmc_dir}
288
289# Set up proxies
290export ftp_proxy=${http_proxy}
291export http_proxy=${http_proxy}
292export https_proxy=${http_proxy}
293
294mkdir -p ${WORKSPACE}/bin
295
296# Configure proxies for BitBake
297if [[ -n "${http_proxy}" ]]; then
298
299  cat > ${WORKSPACE}/bin/git-proxy << \EOF_GIT
300  #!/bin/bash
301  # \$1 = hostname, \$2 = port
302  PROXY=${PROXY_HOST}
303  PROXY_PORT=${PROXY_PORT}
304  exec socat STDIO PROXY:\${PROXY}:\${1}:\${2},proxyport=\${PROXY_PORT}
305EOF_GIT
306
307  chmod a+x ${WORKSPACE}/bin/git-proxy
308  export PATH=${WORKSPACE}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH}
309
310  lock=${HOME}/build-setup.lock
311  flock \${lock} git config --global core.gitProxy ${WORKSPACE}/bin/git-proxy
312  flock \${lock} git config --global http.proxy ${http_proxy}
313
314  flock \${lock} mkdir -p ~/.subversion
315  flock \${lock} cat > ~/.subversion/servers << EOF_SVN
316  [global]
317  http-proxy-host = ${PROXY_HOST}
318  http-proxy-port = ${PROXY_PORT}
319EOF_SVN
320
321  flock \${lock} cat > ~/.wgetrc << EOF_WGETRC
322  https_proxy = ${http_proxy}
323  http_proxy = ${http_proxy}
324  use_proxy = on
325EOF_WGETRC
326
327  flock \${lock} cat > ~/.curlrc << EOF_CURLRC
328  proxy = ${PROXY_HOST}:${PROXY_PORT}
329EOF_CURLRC
330fi
331
332# Source our build env
333${BITBAKE_CMD}
334
335if [[ -z "${MACHINE}" ]]; then
336  echo "MACHINE is not configured for ${target}"
337  exit 1
338fi
339
340export MACHINE="${MACHINE}"
341if [[ -z "${DISTRO}" ]]; then
342  echo "DISTRO is not configured for ${target} so will use default"
343  unset DISTRO
344else
345  export DISTRO="${DISTRO}"
346fi
347
348# bitbake requires SDKMACHINE be x86
349export SDKMACHINE=x86_64
350
351# Custom BitBake config settings
352cat >> conf/local.conf << EOF_CONF
353BB_NUMBER_THREADS = "$num_cpu"
354PARALLEL_MAKE = "-j$num_cpu"
355INHERIT += "rm_work"
356BB_GENERATE_MIRROR_TARBALLS = "1"
357DL_DIR="${ssc_dir}/bitbake_downloads"
358SSTATE_DIR="${ssc_dir}/bitbake_sharedstatecache"
359USER_CLASSES += "buildstats"
360INHERIT:remove = "uninative"
361TMPDIR="${build_dir}"
362${ENV_LOCAL_CONF}
363EOF_CONF
364
365# Kick off a build
366if [[ -n "${nice_priority}" ]]; then
367    nice -${nice_priority} bitbake -k ${BITBAKE_OPTS} ${bitbake_target}
368else
369    bitbake -k ${BITBAKE_OPTS} ${bitbake_target}
370fi
371
372# Copy internal build directory into xtrct_path directory
373if [[ ${xtrct_small_copy_dir} ]]; then
374  mkdir -p ${xtrct_path}/${xtrct_small_copy_dir}
375  timeout ${xtrct_copy_timeout} cp -r ${build_dir}/${xtrct_small_copy_dir}/* ${xtrct_path}/${xtrct_small_copy_dir}
376else
377  timeout ${xtrct_copy_timeout} cp -r ${build_dir}/* ${xtrct_path}
378fi
379
380if [[ 0 -ne $? ]]; then
381  echo "Received a non-zero exit code from timeout"
382  exit 1
383fi
384
385EOF_SCRIPT
386
387chmod a+x "${WORKSPACE}/build.sh"
388
389# Give the Docker image a name based on the distro,tag,arch,and target
390img_name=${img_name:-openbmc/${distro}:${img_tag}-${target}-${ARCH}}
391
392# Ensure appropriate docker build output to see progress and identify
393# any issues
394export BUILDKIT_PROGRESS=plain
395
396# Build the Docker image
397docker build --network=host -t "${img_name}" - <<< "${Dockerfile}"
398
399if [[ "$container_only" = "true" ]]; then
400    exit 0
401fi
402
403# If obmc_dir or ssc_dir are ${HOME} or a subdirectory they will not be mounted
404mount_obmc_dir="-v ""${obmc_dir}"":""${obmc_dir}"" "
405mount_ssc_dir="-v ""${ssc_dir}"":""${ssc_dir}"" "
406mount_workspace_dir="-v ""${WORKSPACE}"":""${WORKSPACE}"" "
407if [[ "${obmc_dir}" = "${HOME}/"* || "${obmc_dir}" = "${HOME}" ]];then
408    mount_obmc_dir=""
409fi
410if [[ "${ssc_dir}" = "${HOME}/"* || "${ssc_dir}" = "${HOME}" ]];then
411    mount_ssc_dir=""
412fi
413if [[ "${WORKSPACE}" = "${HOME}/"* || "${WORKSPACE}" = "${HOME}" ]];then
414    mount_workspace_dir=""
415fi
416
417# If we are building on a podman based machine, need to have this set in
418# the env to allow the home mount to work (no impact on non-podman systems)
419export PODMAN_USERNS="keep-id"
420
421# Run the Docker container, execute the build.sh script
422# shellcheck disable=SC2086 # mount commands word-split purposefully
423docker run \
424    --cap-add=sys_admin \
425    --cap-add=sys_nice \
426    --net=host \
427    --rm=true \
428    -e WORKSPACE="${WORKSPACE}" \
429    -w "${HOME}" \
430    -v "${HOME}:${HOME}" \
431    ${mount_obmc_dir} \
432    ${mount_ssc_dir} \
433    ${mount_workspace_dir} \
434    "${img_name}" \
435    "${WORKSPACE}/build.sh"
436
437# To maintain function of resources that used an older path, add a link
438ln -sf "${xtrct_path}/deploy" "${WORKSPACE}/deploy"
439
440# Timestamp for build
441echo "Build completed, $(date)"
442