xref: /openbmc/openbmc-build-scripts/build-setup.sh (revision 02871c9120ffcdafc42f636a9fbb271b1f5d03df) 
1#!/bin/bash
2###############################################################################
3#
4# This build script is for running the OpenBMC builds as Docker containers.
5#
6###############################################################################
7#
8# Script Variables:
9#  build_scripts_dir  The path of the openbmc-build-scripts directory.
10#                     Default: The directory containing this script
11#  http_proxy         The HTTP address of the proxy server to connect to.
12#                     Default: "", proxy is not setup if this is not set
13#  WORKSPACE          Path of the workspace directory where some intermediate
14#                     files and the images will be saved to.
15#                     Default: "~/{RandomNumber}"
16#  num_cpu            Number of cpu's to give bitbake, default is total amount
17#                     in system
18#  UBUNTU_MIRROR      [optional] The URL of a mirror of Ubuntu to override the
19#                     default ones in /etc/apt/sources.list
20#                     default is empty, and no mirror is used.
21#  ENV_LOCAL_CONF     [optional] The environment variables to inject into the
22#                     build, which will be written into local.conf.
23#                     default is empty.
24#
25# Docker Image Build Variables:
26#  BITBAKE_OPTS       Set to "-c populate_sdk" or whatever other BitBake options
27#                     you'd like to pass into the build.
28#                     Default: "", no options set
29#  build_dir          Path where the actual BitBake build occurs inside the
30#                     container, path cannot be located on network storage.
31#                     Default: "$WORKSPACE/build"
32#  distro             The distro used as the base image for the build image:
33#                     fedora|ubuntu
34#                     Default: "ubuntu"
35#  img_name           The name given to the target build's docker image.
36#                     Default: "openbmc/${distro}:${imgtag}-${target}-${ARCH}"
37#  img_tag            The base docker image distro tag:
38#                     ubuntu: latest|16.04|14.04|trusty|xenial
39#                     fedora: 23|24|25
40#                     Default: "latest"
41#  target             The target we aim to build:
42#                     evb-ast2500|palmetto|qemu|qemux86-64
43#                     romulus|s2600wf|witherspoon|zaius|tiogapass|gsj|mihawk
44#                     witherspoon-tacoma|rainier
45#                     Default: "qemu"
46#  no_tar             Set to true if you do not want the debug tar built
47#                     Default: "false"
48#  nice_priority      Set nice priotity for bitbake command.
49#                     Nice:
50#                       Run with an adjusted niceness, which affects process
51#                       scheduling. Nice values range from -20 (most favorable
52#                       to the process) to 19 (least favorable to the process).
53#                     Default: "", nice is not used if nice_priority is not set
54#
55# Deployment Variables:
56#  obmc_dir           Path of the OpenBMC repo directory used as a reference
57#                     for the build inside the container.
58#                     Default: "${WORKSPACE}/openbmc"
59#  ssc_dir            Path of the OpenBMC shared directory that contains the
60#                     downloads dir and the sstate dir.
61#                     Default: "${HOME}"
62#  xtrct_small_copy_dir
63#                     Directory within build_dir that should be copied to
64#                     xtrct_path. The directory and all parents up to, but not
65#                     including, build_dir will be copied. For example, if
66#                     build_dir is set to "/tmp/openbmc" and this is set to
67#                     "build/tmp", the directory at xtrct_path will have the
68#                     following directory structure:
69#                     xtrct_path
70#                      | - build
71#                        | - tmp
72#                          ...
73#                     Can also be set to the empty string to copy the entire
74#                     contents of build_dir to xtrct_path.
75#                     Default: "deploy/images".
76#
77###############################################################################
78# Trace bash processing. Set -e so when a step fails, we fail the build
79set -xeo pipefail
80
81# Script Variables:
82build_scripts_dir=${build_scripts_dir:-"$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"}
83http_proxy=${http_proxy:-}
84WORKSPACE=${WORKSPACE:-${HOME}/${RANDOM}${RANDOM}}
85num_cpu=${num_cpu:-$(nproc)}
86UBUNTU_MIRROR=${UBUNTU_MIRROR:-""}
87ENV_LOCAL_CONF=${ENV_LOCAL_CONF:-""}
88
89# Docker Image Build Variables:
90build_dir=${build_dir:-${WORKSPACE}/build}
91distro=${distro:-ubuntu}
92img_tag=${img_tag:-latest}
93target=${target:-qemu}
94no_tar=${no_tar:-false}
95nice_priority=${nice_priority:-}
96
97# Deployment variables
98obmc_dir=${obmc_dir:-${WORKSPACE}/openbmc}
99ssc_dir=${ssc_dir:-${HOME}}
100xtrct_small_copy_dir=${xtrct_small_copy_dir:-deploy/images}
101xtrct_path="${obmc_dir}/build/tmp"
102xtrct_copy_timeout="300"
103
104bitbake_target="obmc-phosphor-image"
105PROXY=""
106
107MIRROR=""
108if [[ -n "${UBUNTU_MIRROR}" ]]; then
109    MIRROR="RUN echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME) main restricted universe multiverse\" > /etc/apt/sources.list && \
110        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-updates main restricted universe multiverse\" >> /etc/apt/sources.list && \
111        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-security main restricted universe multiverse\" >> /etc/apt/sources.list && \
112        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-proposed main restricted universe multiverse\" >> /etc/apt/sources.list && \
113        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-backports main restricted universe multiverse\" >> /etc/apt/sources.list"
114fi
115
116# Determine the architecture
117ARCH=$(uname -m)
118
119# Determine the prefix of the Dockerfile's base image
120case ${ARCH} in
121  "ppc64le")
122    DOCKER_BASE="ppc64le/"
123    ;;
124  "x86_64")
125    DOCKER_BASE=""
126    ;;
127  *)
128    echo "Unsupported system architecture(${ARCH}) found for docker image"
129    exit 1
130esac
131
132# Timestamp for job
133echo "Build started, $(date)"
134
135# If the obmc_dir directory doesn't exist clone it in
136if [ ! -d "${obmc_dir}" ]; then
137  echo "Clone in openbmc master to ${obmc_dir}"
138  git clone https://github.com/openbmc/openbmc "${obmc_dir}"
139fi
140
141# Make and chown the xtrct_path directory to avoid permission errors
142if [ ! -d "${xtrct_path}" ]; then
143  mkdir -p "${xtrct_path}"
144fi
145chown "${UID}:${GROUPS[0]}" "${xtrct_path}"
146
147# Work out what build target we should be running and set BitBake command
148MACHINE=""
149case ${target} in
150  palmetto)
151    LAYER_DIR="meta-ibm/meta-palmetto"
152    MACHINE="palmetto"
153    DISTRO="openbmc-openpower"
154    ;;
155  swift)
156    LAYER_DIR="meta-ibm"
157    MACHINE="swift"
158    DISTRO="openbmc-witherspoon"
159    ;;
160  mihawk)
161    LAYER_DIR="meta-ibm"
162    MACHINE="mihawk"
163    DISTRO="openbmc-witherspoon"
164    ;;
165  witherspoon)
166    LAYER_DIR="meta-ibm"
167    MACHINE="witherspoon"
168    DISTRO="openbmc-witherspoon"
169    ;;
170  witherspoon-128)
171    LAYER_DIR="meta-ibm"
172    MACHINE="witherspoon-128"
173    DISTRO="openbmc-witherspoon"
174    ;;
175  witherspoon-tacoma)
176    LAYER_DIR="meta-ibm"
177    MACHINE="witherspoon-tacoma"
178    DISTRO="openbmc-openpower"
179    ;;
180  rainier)
181    LAYER_DIR="meta-ibm"
182    MACHINE="rainier"
183    DISTRO="openbmc-openpower"
184    ;;
185  evb-ast2500)
186    LAYER_DIR="meta-evb/meta-evb-aspeed/meta-evb-ast2500"
187    MACHINE="evb-ast2500"
188    DISTRO="openbmc-phosphor"
189    ;;
190  s2600wf)
191    LAYER_DIR="meta-intel/meta-s2600wf"
192    MACHINE="s2600wf"
193    DISTRO="openbmc-phosphor"
194    ;;
195  zaius)
196    LAYER_DIR="meta-ingrasys/meta-zaius"
197    MACHINE="zaius"
198    DISTRO="openbmc-openpower"
199    ;;
200  romulus)
201    LAYER_DIR="meta-ibm/meta-romulus"
202    MACHINE="romulus"
203    DISTRO="openbmc-openpower"
204    ;;
205  tiogapass)
206    LAYER_DIR="meta-facebook/meta-tiogapass"
207    MACHINE="tiogapass"
208    DISTRO="openbmc-phosphor"
209    ;;
210  gsj)
211    LAYER_DIR="meta-quanta/meta-gsj"
212    MACHINE="gsj"
213    # Use default DISTRO from layer
214    ;;
215  *)
216    exit 1
217    ;;
218esac
219
220BITBAKE_CMD="TEMPLATECONF=${LAYER_DIR}/conf source oe-init-build-env"
221
222# Configure Docker build
223if [[ "${distro}" == fedora ]];then
224
225  if [[ -n "${http_proxy}" ]]; then
226    PROXY="RUN echo \"proxy=${http_proxy}\" >> /etc/dnf/dnf.conf"
227  fi
228
229  Dockerfile=$(cat << EOF
230  FROM ${DOCKER_BASE}${distro}:${img_tag}
231
232  ${PROXY}
233
234  RUN dnf --refresh install -y \
235      bzip2 \
236      chrpath \
237      cpio \
238      diffstat \
239      findutils \
240      gcc \
241      gcc-c++ \
242      git \
243      make \
244      patch \
245      perl-bignum \
246      perl-Data-Dumper \
247      perl-Thread-Queue \
248      python-devel \
249      python3-devel \
250      SDL-devel \
251      socat \
252      subversion \
253      tar \
254      texinfo \
255      wget \
256      which \
257      file \
258      hostname \
259      rpcgen \
260      glibc-langpack-en \
261      glibc-locale-source
262
263  # Set the locale
264  ENV LANG=en_US.utf8
265  RUN localedef -f UTF-8 -i en_US en_US.UTF-8
266
267  RUN grep -q ${GROUPS[0]} /etc/group || groupadd -g ${GROUPS[0]} ${USER}
268  RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS[0]} ${USER}
269
270  USER ${USER}
271  ENV HOME ${HOME}
272  RUN /bin/bash
273EOF
274)
275
276elif [[ "${distro}" == ubuntu ]]; then
277
278  if [[ -n "${http_proxy}" ]]; then
279    PROXY="RUN echo \"Acquire::http::Proxy \\"\"${http_proxy}/\\"\";\" > /etc/apt/apt.conf.d/000apt-cacher-ng-proxy"
280  fi
281
282  Dockerfile=$(cat << EOF
283  FROM ${DOCKER_BASE}${distro}:${img_tag}
284
285  ${PROXY}
286  ${MIRROR}
287
288  ENV DEBIAN_FRONTEND noninteractive
289
290  RUN apt-get update && apt-get install -yy \
291      build-essential \
292      chrpath \
293      debianutils \
294      diffstat \
295      gawk \
296      git \
297      libdata-dumper-simple-perl \
298      libsdl1.2-dev \
299      libthread-queue-any-perl \
300      locales \
301      python \
302      python3 \
303      socat \
304      subversion \
305      texinfo \
306      cpio \
307      wget \
308      iputils-ping
309
310  # Set the locale
311  RUN locale-gen en_US.UTF-8
312  ENV LANG en_US.UTF-8
313  ENV LANGUAGE en_US:en
314  ENV LC_ALL en_US.UTF-8
315
316  RUN grep -q ${GROUPS[0]} /etc/group || groupadd -g ${GROUPS[0]} ${USER}
317  RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS[0]} ${USER}
318
319  USER ${USER}
320  ENV HOME ${HOME}
321  RUN /bin/bash
322EOF
323)
324fi
325
326# Create the Docker run script
327export PROXY_HOST=${http_proxy/#http*:\/\/}
328export PROXY_HOST=${PROXY_HOST/%:[0-9]*}
329export PROXY_PORT=${http_proxy/#http*:\/\/*:}
330
331mkdir -p "${WORKSPACE}"
332
333# Determine command for bitbake image build
334if [ "$no_tar" = "false" ]; then
335    bitbake_target="${bitbake_target} obmc-phosphor-debug-tarball"
336fi
337
338cat > "${WORKSPACE}"/build.sh << EOF_SCRIPT
339#!/bin/bash
340
341set -xeo pipefail
342
343# Go into the OpenBMC directory, the build will handle changing directories
344cd ${obmc_dir}
345
346# Set up proxies
347export ftp_proxy=${http_proxy}
348export http_proxy=${http_proxy}
349export https_proxy=${http_proxy}
350
351mkdir -p ${WORKSPACE}/bin
352
353# Configure proxies for BitBake
354if [[ -n "${http_proxy}" ]]; then
355
356  cat > ${WORKSPACE}/bin/git-proxy << \EOF_GIT
357  #!/bin/bash
358  # \$1 = hostname, \$2 = port
359  PROXY=${PROXY_HOST}
360  PROXY_PORT=${PROXY_PORT}
361  exec socat STDIO PROXY:\${PROXY}:\${1}:\${2},proxyport=\${PROXY_PORT}
362EOF_GIT
363
364  chmod a+x ${WORKSPACE}/bin/git-proxy
365  export PATH=${WORKSPACE}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH}
366
367  lock=${HOME}/build-setup.lock
368  flock \${lock} git config --global core.gitProxy ${WORKSPACE}/bin/git-proxy
369  flock \${lock} git config --global http.proxy ${http_proxy}
370
371  flock \${lock} mkdir -p ~/.subversion
372  flock \${lock} cat > ~/.subversion/servers << EOF_SVN
373  [global]
374  http-proxy-host = ${PROXY_HOST}
375  http-proxy-port = ${PROXY_PORT}
376EOF_SVN
377
378  flock \${lock} cat > ~/.wgetrc << EOF_WGETRC
379  https_proxy = ${http_proxy}
380  http_proxy = ${http_proxy}
381  use_proxy = on
382EOF_WGETRC
383
384  flock \${lock} cat > ~/.curlrc << EOF_CURLRC
385  proxy = ${PROXY_HOST}:${PROXY_PORT}
386EOF_CURLRC
387fi
388
389# Source our build env
390${BITBAKE_CMD}
391
392if [[ -z "${MACHINE}" ]]; then
393  echo "MACHINE is not configured for ${target}"
394  exit 1
395fi
396
397export MACHINE="${MACHINE}"
398if [[ -z "${DISTRO}" ]]; then
399  echo "DISTRO is not configured for ${target} so will use default"
400  unset DISTRO
401else
402  export DISTRO="${DISTRO}"
403fi
404
405# bitbake requires SDKMACHINE be x86
406export SDKMACHINE=x86_64
407
408# Custom BitBake config settings
409cat >> conf/local.conf << EOF_CONF
410BB_NUMBER_THREADS = "$(nproc)"
411PARALLEL_MAKE = "-j$(nproc)"
412INHERIT += "rm_work"
413BB_GENERATE_MIRROR_TARBALLS = "1"
414DL_DIR="${ssc_dir}/bitbake_downloads"
415SSTATE_DIR="${ssc_dir}/bitbake_sharedstatecache"
416USER_CLASSES += "buildstats"
417INHERIT_remove = "uninative"
418TMPDIR="${build_dir}"
419${ENV_LOCAL_CONF}
420EOF_CONF
421
422# Kick off a build
423if [[ -n "${nice_priority}" ]]; then
424    nice -${nice_priority} bitbake ${BITBAKE_OPTS} ${bitbake_target}
425else
426    bitbake ${BITBAKE_OPTS} ${bitbake_target}
427fi
428
429# Copy internal build directory into xtrct_path directory
430if [[ ${xtrct_small_copy_dir} ]]; then
431  mkdir -p ${xtrct_path}/${xtrct_small_copy_dir}
432  timeout ${xtrct_copy_timeout} cp -r ${build_dir}/${xtrct_small_copy_dir}/* ${xtrct_path}/${xtrct_small_copy_dir}
433else
434  timeout ${xtrct_copy_timeout} cp -r ${build_dir}/* ${xtrct_path}
435fi
436
437if [[ 0 -ne $? ]]; then
438  echo "Received a non-zero exit code from timeout"
439  exit 1
440fi
441
442EOF_SCRIPT
443
444chmod a+x "${WORKSPACE}/build.sh"
445
446# Give the Docker image a name based on the distro,tag,arch,and target
447img_name=${img_name:-openbmc/${distro}:${img_tag}-${target}-${ARCH}}
448
449# Build the Docker image
450docker build -t "${img_name}" - <<< "${Dockerfile}"
451
452# If obmc_dir or ssc_dir are ${HOME} or a subdirectory they will not be mounted
453mount_obmc_dir="-v ""${obmc_dir}"":""${obmc_dir}"" "
454mount_ssc_dir="-v ""${ssc_dir}"":""${ssc_dir}"" "
455mount_workspace_dir="-v ""${WORKSPACE}"":""${WORKSPACE}"" "
456if [[ "${obmc_dir}" = "${HOME}/"* || "${obmc_dir}" = "${HOME}" ]];then
457mount_obmc_dir=""
458fi
459if [[ "${ssc_dir}" = "${HOME}/"* || "${ssc_dir}" = "${HOME}" ]];then
460mount_ssc_dir=""
461fi
462if [[ "${WORKSPACE}" = "${HOME}/"* || "${WORKSPACE}" = "${HOME}" ]];then
463mount_workspace_dir=""
464fi
465
466# Run the Docker container, execute the build.sh script
467# shellcheck disable=SC2086 # mount commands word-split purposefully
468docker run \
469--cap-add=sys_admin \
470--cap-add=sys_nice \
471--net=host \
472--rm=true \
473-e WORKSPACE="${WORKSPACE}" \
474-w "${HOME}" \
475-v "${HOME}:${HOME}" \
476${mount_obmc_dir} \
477${mount_ssc_dir} \
478${mount_workspace_dir} \
479--cpus="$num_cpu" \
480"${img_name}" \
481"${WORKSPACE}/build.sh"
482
483# To maintain function of resources that used an older path, add a link
484ln -sf "${xtrct_path}/deploy" "${WORKSPACE}/deploy"
485
486# Timestamp for build
487echo "Build completed, $(date)"
488