1#!/bin/bash
2###############################################################################
3#
4# This build script is for running the OpenBMC builds as Docker containers.
5#
6###############################################################################
7#
8# Script Variables:
9#  build_scripts_dir  The path of the openbmc-build-scripts directory.
10#                     Default: The directory containing this script
11#  http_proxy         The HTTP address of the proxy server to connect to.
12#                     Default: "", proxy is not setup if this is not set
13#  WORKSPACE          Path of the workspace directory where some intermediate
14#                     files and the images will be saved to.
15#                     Default: "~/{RandomNumber}"
16#  num_cpu            Number of cpu's to give bitbake, default is total amount
17#                     in system
18#  UBUNTU_MIRROR      [optional] The URL of a mirror of Ubuntu to override the
19#                     default ones in /etc/apt/sources.list
20#                     default is empty, and no mirror is used.
21#  ENV_LOCAL_CONF     [optional] The environment variables to inject into the
22#                     build, which will be written into local.conf.
23#                     default is empty.
24#
25# Docker Image Build Variables:
26#  BITBAKE_OPTS       Set to "-c populate_sdk" or whatever other BitBake options
27#                     you'd like to pass into the build.
28#                     Default: "", no options set
29#  build_dir          Path where the actual BitBake build occurs inside the
30#                     container, path cannot be located on network storage.
31#                     Default: "$WORKSPACE/build"
32#  distro             The distro used as the base image for the build image:
33#                     fedora|ubuntu
34#                     Default: "ubuntu"
35#  img_name           The name given to the target build's docker image.
36#                     Default: "openbmc/${distro}:${imgtag}-${target}-${ARCH}"
37#  img_tag            The base docker image distro tag:
38#                     ubuntu: latest|16.04|14.04|trusty|xenial
39#                     fedora: 23|24|25
40#                     Default: "latest"
41#  target             The target we aim to build.  Any system supported by
42#                     the openbmc/openbmc `setup` script is an option.
43#                     Default: "qemuarm"
44#  no_tar             Set to true if you do not want the debug tar built
45#                     Default: "false"
46#  nice_priority      Set nice priotity for bitbake command.
47#                     Nice:
48#                       Run with an adjusted niceness, which affects process
49#                       scheduling. Nice values range from -20 (most favorable
50#                       to the process) to 19 (least favorable to the process).
51#                     Default: "", nice is not used if nice_priority is not set
52#
53# Deployment Variables:
54#  obmc_dir           Path of the OpenBMC repo directory used as a reference
55#                     for the build inside the container.
56#                     Default: "${WORKSPACE}/openbmc"
57#  ssc_dir            Path of the OpenBMC shared directory that contains the
58#                     downloads dir and the sstate dir.
59#                     Default: "${HOME}"
60#  xtrct_small_copy_dir
61#                     Directory within build_dir that should be copied to
62#                     xtrct_path. The directory and all parents up to, but not
63#                     including, build_dir will be copied. For example, if
64#                     build_dir is set to "/tmp/openbmc" and this is set to
65#                     "build/tmp", the directory at xtrct_path will have the
66#                     following directory structure:
67#                     xtrct_path
68#                      | - build
69#                        | - tmp
70#                          ...
71#                     Can also be set to the empty string to copy the entire
72#                     contents of build_dir to xtrct_path.
73#                     Default: "deploy/images".
74#
75###############################################################################
76# Trace bash processing. Set -e so when a step fails, we fail the build
77set -xeo pipefail
78
79# Script Variables:
80build_scripts_dir=${build_scripts_dir:-"$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"}
81http_proxy=${http_proxy:-}
82WORKSPACE=${WORKSPACE:-${HOME}/${RANDOM}${RANDOM}}
83num_cpu=${num_cpu:-$(nproc)}
84UBUNTU_MIRROR=${UBUNTU_MIRROR:-""}
85ENV_LOCAL_CONF=${ENV_LOCAL_CONF:-""}
86
87# Docker Image Build Variables:
88build_dir=${build_dir:-${WORKSPACE}/build}
89distro=${distro:-ubuntu}
90img_tag=${img_tag:-latest}
91target=${target:-qemuarm}
92no_tar=${no_tar:-false}
93nice_priority=${nice_priority:-}
94
95# Deployment variables
96obmc_dir=${obmc_dir:-${WORKSPACE}/openbmc}
97ssc_dir=${ssc_dir:-${HOME}}
98xtrct_small_copy_dir=${xtrct_small_copy_dir:-deploy/images}
99xtrct_path="${obmc_dir}/build/tmp"
100xtrct_copy_timeout="300"
101
102bitbake_target="obmc-phosphor-image"
103PROXY=""
104
105MIRROR=""
106if [[ -n "${UBUNTU_MIRROR}" ]]; then
107    MIRROR="RUN echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME) main restricted universe multiverse\" > /etc/apt/sources.list && \
108        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-updates main restricted universe multiverse\" >> /etc/apt/sources.list && \
109        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-security main restricted universe multiverse\" >> /etc/apt/sources.list && \
110        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-proposed main restricted universe multiverse\" >> /etc/apt/sources.list && \
111        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-backports main restricted universe multiverse\" >> /etc/apt/sources.list"
112fi
113
114# Determine the architecture
115ARCH=$(uname -m)
116
117# Determine the prefix of the Dockerfile's base image
118case ${ARCH} in
119  "ppc64le")
120    DOCKER_BASE="ppc64le/"
121    ;;
122  "x86_64")
123    DOCKER_BASE=""
124    ;;
125  *)
126    echo "Unsupported system architecture(${ARCH}) found for docker image"
127    exit 1
128esac
129
130# Timestamp for job
131echo "Build started, $(date)"
132
133# If the obmc_dir directory doesn't exist clone it in
134if [ ! -d "${obmc_dir}" ]; then
135  echo "Clone in openbmc master to ${obmc_dir}"
136  git clone https://github.com/openbmc/openbmc "${obmc_dir}"
137fi
138
139# Make and chown the xtrct_path directory to avoid permission errors
140if [ ! -d "${xtrct_path}" ]; then
141  mkdir -p "${xtrct_path}"
142fi
143chown "${UID}:${GROUPS[0]}" "${xtrct_path}"
144
145# Perform overrides for specific machines as required.
146DISTRO=${DISTRO:-}
147
148# Set build target and BitBake command
149MACHINE="${target}"
150BITBAKE_CMD="source ./setup ${MACHINE} ${build_dir}"
151
152# Configure Docker build
153if [[ "${distro}" == fedora ]];then
154
155  if [[ -n "${http_proxy}" ]]; then
156    PROXY="RUN echo \"proxy=${http_proxy}\" >> /etc/dnf/dnf.conf"
157  fi
158
159  Dockerfile=$(cat << EOF
160  FROM ${DOCKER_BASE}${distro}:${img_tag}
161
162  ${PROXY}
163
164  RUN dnf --refresh install -y \
165      bzip2 \
166      chrpath \
167      cpio \
168      diffstat \
169      findutils \
170      gcc \
171      gcc-c++ \
172      git \
173      make \
174      patch \
175      perl-bignum \
176      perl-Data-Dumper \
177      perl-Thread-Queue \
178      python-devel \
179      python3-devel \
180      SDL-devel \
181      socat \
182      subversion \
183      tar \
184      texinfo \
185      wget \
186      which \
187      file \
188      hostname \
189      rpcgen \
190      glibc-langpack-en \
191      glibc-locale-source
192
193  # Set the locale
194  ENV LANG=en_US.utf8
195  RUN localedef -f UTF-8 -i en_US en_US.UTF-8
196
197  RUN grep -q ${GROUPS[0]} /etc/group || groupadd -g ${GROUPS[0]} ${USER}
198  RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS[0]} ${USER}
199
200  USER ${USER}
201  ENV HOME ${HOME}
202  RUN /bin/bash
203EOF
204)
205
206elif [[ "${distro}" == ubuntu ]]; then
207
208  if [[ -n "${http_proxy}" ]]; then
209    PROXY="RUN echo \"Acquire::http::Proxy \\"\"${http_proxy}/\\"\";\" > /etc/apt/apt.conf.d/000apt-cacher-ng-proxy"
210  fi
211
212  Dockerfile=$(cat << EOF
213  FROM ${DOCKER_BASE}${distro}:${img_tag}
214
215  ${PROXY}
216  ${MIRROR}
217
218  ENV DEBIAN_FRONTEND noninteractive
219
220  RUN apt-get update && apt-get install -yy \
221      build-essential \
222      chrpath \
223      cpio \
224      debianutils \
225      diffstat \
226      gawk \
227      git \
228      iputils-ping \
229      libdata-dumper-simple-perl \
230      liblz4-tool \
231      libsdl1.2-dev \
232      libthread-queue-any-perl \
233      locales \
234      python \
235      python3 \
236      socat \
237      subversion \
238      texinfo \
239      wget \
240      zstd
241
242  # Set the locale
243  RUN locale-gen en_US.UTF-8
244  ENV LANG en_US.UTF-8
245  ENV LANGUAGE en_US:en
246  ENV LC_ALL en_US.UTF-8
247
248  RUN grep -q ${GROUPS[0]} /etc/group || groupadd -g ${GROUPS[0]} ${USER}
249  RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS[0]} ${USER}
250
251  USER ${USER}
252  ENV HOME ${HOME}
253  RUN /bin/bash
254EOF
255)
256fi
257
258# Create the Docker run script
259export PROXY_HOST=${http_proxy/#http*:\/\/}
260export PROXY_HOST=${PROXY_HOST/%:[0-9]*}
261export PROXY_PORT=${http_proxy/#http*:\/\/*:}
262
263mkdir -p "${WORKSPACE}"
264
265# Determine command for bitbake image build
266if [ "$no_tar" = "false" ]; then
267    bitbake_target="${bitbake_target} obmc-phosphor-debug-tarball"
268fi
269
270cat > "${WORKSPACE}"/build.sh << EOF_SCRIPT
271#!/bin/bash
272
273set -xeo pipefail
274
275# Go into the OpenBMC directory, the build will handle changing directories
276cd ${obmc_dir}
277
278# Set up proxies
279export ftp_proxy=${http_proxy}
280export http_proxy=${http_proxy}
281export https_proxy=${http_proxy}
282
283mkdir -p ${WORKSPACE}/bin
284
285# Configure proxies for BitBake
286if [[ -n "${http_proxy}" ]]; then
287
288  cat > ${WORKSPACE}/bin/git-proxy << \EOF_GIT
289  #!/bin/bash
290  # \$1 = hostname, \$2 = port
291  PROXY=${PROXY_HOST}
292  PROXY_PORT=${PROXY_PORT}
293  exec socat STDIO PROXY:\${PROXY}:\${1}:\${2},proxyport=\${PROXY_PORT}
294EOF_GIT
295
296  chmod a+x ${WORKSPACE}/bin/git-proxy
297  export PATH=${WORKSPACE}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH}
298
299  lock=${HOME}/build-setup.lock
300  flock \${lock} git config --global core.gitProxy ${WORKSPACE}/bin/git-proxy
301  flock \${lock} git config --global http.proxy ${http_proxy}
302
303  flock \${lock} mkdir -p ~/.subversion
304  flock \${lock} cat > ~/.subversion/servers << EOF_SVN
305  [global]
306  http-proxy-host = ${PROXY_HOST}
307  http-proxy-port = ${PROXY_PORT}
308EOF_SVN
309
310  flock \${lock} cat > ~/.wgetrc << EOF_WGETRC
311  https_proxy = ${http_proxy}
312  http_proxy = ${http_proxy}
313  use_proxy = on
314EOF_WGETRC
315
316  flock \${lock} cat > ~/.curlrc << EOF_CURLRC
317  proxy = ${PROXY_HOST}:${PROXY_PORT}
318EOF_CURLRC
319fi
320
321# Source our build env
322${BITBAKE_CMD}
323
324if [[ -z "${MACHINE}" ]]; then
325  echo "MACHINE is not configured for ${target}"
326  exit 1
327fi
328
329export MACHINE="${MACHINE}"
330if [[ -z "${DISTRO}" ]]; then
331  echo "DISTRO is not configured for ${target} so will use default"
332  unset DISTRO
333else
334  export DISTRO="${DISTRO}"
335fi
336
337# bitbake requires SDKMACHINE be x86
338export SDKMACHINE=x86_64
339
340# Custom BitBake config settings
341cat >> conf/local.conf << EOF_CONF
342BB_NUMBER_THREADS = "$(nproc)"
343PARALLEL_MAKE = "-j$(nproc)"
344INHERIT += "rm_work"
345BB_GENERATE_MIRROR_TARBALLS = "1"
346DL_DIR="${ssc_dir}/bitbake_downloads"
347SSTATE_DIR="${ssc_dir}/bitbake_sharedstatecache"
348USER_CLASSES += "buildstats"
349INHERIT:remove = "uninative"
350TMPDIR="${build_dir}"
351${ENV_LOCAL_CONF}
352EOF_CONF
353
354# Kick off a build
355if [[ -n "${nice_priority}" ]]; then
356    nice -${nice_priority} bitbake ${BITBAKE_OPTS} ${bitbake_target}
357else
358    bitbake ${BITBAKE_OPTS} ${bitbake_target}
359fi
360
361# Copy internal build directory into xtrct_path directory
362if [[ ${xtrct_small_copy_dir} ]]; then
363  mkdir -p ${xtrct_path}/${xtrct_small_copy_dir}
364  timeout ${xtrct_copy_timeout} cp -r ${build_dir}/${xtrct_small_copy_dir}/* ${xtrct_path}/${xtrct_small_copy_dir}
365else
366  timeout ${xtrct_copy_timeout} cp -r ${build_dir}/* ${xtrct_path}
367fi
368
369if [[ 0 -ne $? ]]; then
370  echo "Received a non-zero exit code from timeout"
371  exit 1
372fi
373
374EOF_SCRIPT
375
376chmod a+x "${WORKSPACE}/build.sh"
377
378# Give the Docker image a name based on the distro,tag,arch,and target
379img_name=${img_name:-openbmc/${distro}:${img_tag}-${target}-${ARCH}}
380
381# Build the Docker image
382docker build -t "${img_name}" - <<< "${Dockerfile}"
383
384# If obmc_dir or ssc_dir are ${HOME} or a subdirectory they will not be mounted
385mount_obmc_dir="-v ""${obmc_dir}"":""${obmc_dir}"" "
386mount_ssc_dir="-v ""${ssc_dir}"":""${ssc_dir}"" "
387mount_workspace_dir="-v ""${WORKSPACE}"":""${WORKSPACE}"" "
388if [[ "${obmc_dir}" = "${HOME}/"* || "${obmc_dir}" = "${HOME}" ]];then
389mount_obmc_dir=""
390fi
391if [[ "${ssc_dir}" = "${HOME}/"* || "${ssc_dir}" = "${HOME}" ]];then
392mount_ssc_dir=""
393fi
394if [[ "${WORKSPACE}" = "${HOME}/"* || "${WORKSPACE}" = "${HOME}" ]];then
395mount_workspace_dir=""
396fi
397
398# Run the Docker container, execute the build.sh script
399# shellcheck disable=SC2086 # mount commands word-split purposefully
400docker run \
401--cap-add=sys_admin \
402--cap-add=sys_nice \
403--net=host \
404--rm=true \
405-e WORKSPACE="${WORKSPACE}" \
406-w "${HOME}" \
407-v "${HOME}:${HOME}" \
408${mount_obmc_dir} \
409${mount_ssc_dir} \
410${mount_workspace_dir} \
411--cpus="$num_cpu" \
412"${img_name}" \
413"${WORKSPACE}/build.sh"
414
415# To maintain function of resources that used an older path, add a link
416ln -sf "${xtrct_path}/deploy" "${WORKSPACE}/deploy"
417
418# Timestamp for build
419echo "Build completed, $(date)"
420