xref: /openbmc/openbmc-build-scripts/build-setup.sh (revision 8448cfd1873e3a8346738aeaeb11f77c6b3778e3) 
1#!/bin/bash
2###############################################################################
3#
4# This build script is for running the OpenBMC builds as Docker containers.
5#
6###############################################################################
7#
8# Script Variables:
9#  build_scripts_dir  The path of the openbmc-build-scripts directory.
10#                     Default: The directory containing this script
11#  http_proxy         The HTTP address of the proxy server to connect to.
12#                     Default: "", proxy is not setup if this is not set
13#  WORKSPACE          Path of the workspace directory where some intermediate
14#                     files and the images will be saved to.
15#                     Default: "~/{RandomNumber}"
16#  num_cpu            Number of cpu's to give bitbake, default is total amount
17#                     in system
18#  UBUNTU_MIRROR:     <optional, the URL of a mirror of Ubuntu to override the
19#                     default ones in /etc/apt/sources.list>
20#                     default is empty, and no mirror is used.
21#
22# Docker Image Build Variables:
23#  BITBAKE_OPTS       Set to "-c populate_sdk" or whatever other BitBake options
24#                     you'd like to pass into the build.
25#                     Default: "", no options set
26#  build_dir          Path where the actual BitBake build occurs inside the
27#                     container, path cannot be located on network storage.
28#                     Default: "$WORKSPACE/build"
29#  distro             The distro used as the base image for the build image:
30#                     fedora|ubuntu
31#                     Default: "ubuntu"
32#  img_name           The name given to the target build's docker image.
33#                     Default: "openbmc/${distro}:${imgtag}-${target}-${ARCH}"
34#  img_tag            The base docker image distro tag:
35#                     ubuntu: latest|16.04|14.04|trusty|xenial
36#                     fedora: 23|24|25
37#                     Default: "latest"
38#  target             The target we aim to build:
39#                     evb-ast2500|palmetto|qemu|qemux86-64
40#                     romulus|s2600wf|witherspoon|zaius|tiogapass|gsj|mihawk
41#                     witherspoon-tacoma|rainier
42#                     Default: "qemu"
43#  no_tar             Set to true if you do not want the debug tar built
44#                     Default: "false"
45#  nice_priority      Set nice priotity for bitbake command.
46#                     Nice:
47#                       Run with an adjusted niceness, which affects process
48#                       scheduling. Nice values range from -20 (most favorable
49#                       to the process) to 19 (least favorable to the process).
50#                     Default: "", nice is not used if nice_priority is not set
51#
52# Deployment Variables:
53#  obmc_dir           Path of the OpenBMC repo directory used as a reference
54#                     for the build inside the container.
55#                     Default: "${WORKSPACE}/openbmc"
56#  ssc_dir            Path of the OpenBMC shared directory that contains the
57#                     downloads dir and the sstate dir.
58#                     Default: "${HOME}"
59#  xtrct_small_copy_dir
60#                     Directory within build_dir that should be copied to
61#                     xtrct_path. The directory and all parents up to, but not
62#                     including, build_dir will be copied. For example, if
63#                     build_dir is set to "/tmp/openbmc" and this is set to
64#                     "build/tmp", the directory at xtrct_path will have the
65#                     following directory structure:
66#                     xtrct_path
67#                      | - build
68#                        | - tmp
69#                          ...
70#                     Can also be set to the empty string to copy the entire
71#                     contents of build_dir to xtrct_path.
72#                     Default: "deploy/images".
73#
74###############################################################################
75# Trace bash processing. Set -e so when a step fails, we fail the build
76set -xeo pipefail
77
78# Script Variables:
79build_scripts_dir=${build_scripts_dir:-"$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"}
80http_proxy=${http_proxy:-}
81WORKSPACE=${WORKSPACE:-${HOME}/${RANDOM}${RANDOM}}
82num_cpu=${num_cpu:-$(nproc)}
83UBUNTU_MIRROR=${UBUNTU_MIRROR:-""}
84
85# Docker Image Build Variables:
86build_dir=${build_dir:-${WORKSPACE}/build}
87distro=${distro:-ubuntu}
88img_tag=${img_tag:-latest}
89target=${target:-qemu}
90no_tar=${no_tar:-false}
91nice_priority=${nice_priority:-}
92
93# Deployment variables
94obmc_dir=${obmc_dir:-${WORKSPACE}/openbmc}
95ssc_dir=${ssc_dir:-${HOME}}
96xtrct_small_copy_dir=${xtrct_small_copy_dir:-deploy/images}
97xtrct_path="${obmc_dir}/build/tmp"
98xtrct_copy_timeout="300"
99
100bitbake_target="obmc-phosphor-image"
101PROXY=""
102
103MIRROR=""
104if [[ -n "${UBUNTU_MIRROR}" ]]; then
105    MIRROR="RUN echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME) main restricted universe multiverse\" > /etc/apt/sources.list && \
106        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-updates main restricted universe multiverse\" >> /etc/apt/sources.list && \
107        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-security main restricted universe multiverse\" >> /etc/apt/sources.list && \
108        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-proposed main restricted universe multiverse\" >> /etc/apt/sources.list && \
109        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-backports main restricted universe multiverse\" >> /etc/apt/sources.list"
110fi
111
112# Determine the architecture
113ARCH=$(uname -m)
114
115# Determine the prefix of the Dockerfile's base image
116case ${ARCH} in
117  "ppc64le")
118    DOCKER_BASE="ppc64le/"
119    ;;
120  "x86_64")
121    DOCKER_BASE=""
122    ;;
123  *)
124    echo "Unsupported system architecture(${ARCH}) found for docker image"
125    exit 1
126esac
127
128# Timestamp for job
129echo "Build started, $(date)"
130
131# If the obmc_dir directory doesn't exist clone it in
132if [ ! -d ${obmc_dir} ]; then
133  echo "Clone in openbmc master to ${obmc_dir}"
134  git clone https://github.com/openbmc/openbmc ${obmc_dir}
135fi
136
137# Make and chown the xtrct_path directory to avoid permission errors
138if [ ! -d ${xtrct_path} ]; then
139  mkdir -p ${xtrct_path}
140fi
141chown ${UID}:${GROUPS} ${xtrct_path}
142
143# Work out what build target we should be running and set BitBake command
144MACHINE=""
145case ${target} in
146  palmetto)
147    LAYER_DIR="meta-ibm/meta-palmetto"
148    MACHINE="palmetto"
149    DISTRO="openbmc-openpower"
150    ;;
151  swift)
152    LAYER_DIR="meta-ibm"
153    MACHINE="swift"
154    DISTRO="openbmc-witherspoon"
155    ;;
156  mihawk)
157    LAYER_DIR="meta-ibm"
158    MACHINE="mihawk"
159    DISTRO="openbmc-witherspoon"
160    ;;
161  witherspoon)
162    LAYER_DIR="meta-ibm"
163    MACHINE="witherspoon"
164    DISTRO="openbmc-witherspoon"
165    ;;
166  witherspoon-128)
167    LAYER_DIR="meta-ibm"
168    MACHINE="witherspoon-128"
169    DISTRO="openbmc-witherspoon"
170    ;;
171  witherspoon-tacoma)
172    LAYER_DIR="meta-ibm"
173    MACHINE="witherspoon-tacoma"
174    DISTRO="openbmc-openpower"
175    ;;
176  rainier)
177    LAYER_DIR="meta-ibm"
178    MACHINE="rainier"
179    DISTRO="openbmc-openpower"
180    ;;
181  evb-ast2500)
182    LAYER_DIR="meta-evb/meta-evb-aspeed/meta-evb-ast2500"
183    MACHINE="evb-ast2500"
184    DISTRO="openbmc-phosphor"
185    ;;
186  s2600wf)
187    LAYER_DIR="meta-intel/meta-s2600wf"
188    MACHINE="s2600wf"
189    DISTRO="openbmc-phosphor"
190    ;;
191  zaius)
192    LAYER_DIR="meta-ingrasys/meta-zaius"
193    MACHINE="zaius"
194    DISTRO="openbmc-openpower"
195    ;;
196  romulus)
197    LAYER_DIR="meta-ibm/meta-romulus"
198    MACHINE="romulus"
199    DISTRO="openbmc-openpower"
200    ;;
201  tiogapass)
202    LAYER_DIR="meta-facebook/meta-tiogapass"
203    MACHINE="tiogapass"
204    DISTRO="openbmc-phosphor"
205    ;;
206  gsj)
207    LAYER_DIR="meta-quanta/meta-gsj"
208    MACHINE="gsj"
209    # Use default DISTRO from layer
210    ;;
211  *)
212    exit 1
213    ;;
214esac
215
216BITBAKE_CMD="TEMPLATECONF=${LAYER_DIR}/conf source oe-init-build-env"
217
218# Configure Docker build
219if [[ "${distro}" == fedora ]];then
220
221  if [[ -n "${http_proxy}" ]]; then
222    PROXY="RUN echo \"proxy=${http_proxy}\" >> /etc/dnf/dnf.conf"
223  fi
224
225  Dockerfile=$(cat << EOF
226  FROM ${DOCKER_BASE}${distro}:${img_tag}
227
228  ${PROXY}
229
230  RUN dnf --refresh install -y \
231      bzip2 \
232      chrpath \
233      cpio \
234      diffstat \
235      findutils \
236      gcc \
237      gcc-c++ \
238      git \
239      make \
240      patch \
241      perl-bignum \
242      perl-Data-Dumper \
243      perl-Thread-Queue \
244      python-devel \
245      python3-devel \
246      SDL-devel \
247      socat \
248      subversion \
249      tar \
250      texinfo \
251      wget \
252      which \
253      file \
254      hostname \
255      rpcgen \
256      glibc-langpack-en \
257      glibc-locale-source
258
259  # Set the locale
260  ENV LANG=en_US.utf8
261  RUN localedef -f UTF-8 -i en_US en_US.UTF-8
262
263  RUN grep -q ${GROUPS} /etc/group || groupadd -g ${GROUPS} ${USER}
264  RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS} ${USER}
265
266  USER ${USER}
267  ENV HOME ${HOME}
268  RUN /bin/bash
269EOF
270)
271
272elif [[ "${distro}" == ubuntu ]]; then
273
274  if [[ -n "${http_proxy}" ]]; then
275    PROXY="RUN echo \"Acquire::http::Proxy \\"\"${http_proxy}/\\"\";\" > /etc/apt/apt.conf.d/000apt-cacher-ng-proxy"
276  fi
277
278  Dockerfile=$(cat << EOF
279  FROM ${DOCKER_BASE}${distro}:${img_tag}
280
281  ${PROXY}
282  ${MIRROR}
283
284  ENV DEBIAN_FRONTEND noninteractive
285
286  RUN apt-get update && apt-get install -yy \
287      build-essential \
288      chrpath \
289      debianutils \
290      diffstat \
291      gawk \
292      git \
293      libdata-dumper-simple-perl \
294      libsdl1.2-dev \
295      libthread-queue-any-perl \
296      locales \
297      python \
298      python3 \
299      socat \
300      subversion \
301      texinfo \
302      cpio \
303      wget \
304      iputils-ping
305
306  # Set the locale
307  RUN locale-gen en_US.UTF-8
308  ENV LANG en_US.UTF-8
309  ENV LANGUAGE en_US:en
310  ENV LC_ALL en_US.UTF-8
311
312  RUN grep -q ${GROUPS} /etc/group || groupadd -g ${GROUPS} ${USER}
313  RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS} ${USER}
314
315  USER ${USER}
316  ENV HOME ${HOME}
317  RUN /bin/bash
318EOF
319)
320fi
321
322# Create the Docker run script
323export PROXY_HOST=${http_proxy/#http*:\/\/}
324export PROXY_HOST=${PROXY_HOST/%:[0-9]*}
325export PROXY_PORT=${http_proxy/#http*:\/\/*:}
326
327mkdir -p ${WORKSPACE}
328
329# Determine command for bitbake image build
330if [ $no_tar = "false" ]; then
331    bitbake_target="${bitbake_target} obmc-phosphor-debug-tarball"
332fi
333
334cat > "${WORKSPACE}"/build.sh << EOF_SCRIPT
335#!/bin/bash
336
337set -xeo pipefail
338
339# Go into the OpenBMC directory, the build will handle changing directories
340cd ${obmc_dir}
341
342# Set up proxies
343export ftp_proxy=${http_proxy}
344export http_proxy=${http_proxy}
345export https_proxy=${http_proxy}
346
347mkdir -p ${WORKSPACE}/bin
348
349# Configure proxies for BitBake
350if [[ -n "${http_proxy}" ]]; then
351
352  cat > ${WORKSPACE}/bin/git-proxy << \EOF_GIT
353  #!/bin/bash
354  # \$1 = hostname, \$2 = port
355  PROXY=${PROXY_HOST}
356  PROXY_PORT=${PROXY_PORT}
357  exec socat STDIO PROXY:\${PROXY}:\${1}:\${2},proxyport=\${PROXY_PORT}
358EOF_GIT
359
360  chmod a+x ${WORKSPACE}/bin/git-proxy
361  export PATH=${WORKSPACE}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH}
362
363  lock=${HOME}/build-setup.lock
364  flock \${lock} git config --global core.gitProxy ${WORKSPACE}/bin/git-proxy
365  flock \${lock} git config --global http.proxy ${http_proxy}
366
367  flock \${lock} mkdir -p ~/.subversion
368  flock \${lock} cat > ~/.subversion/servers << EOF_SVN
369  [global]
370  http-proxy-host = ${PROXY_HOST}
371  http-proxy-port = ${PROXY_PORT}
372EOF_SVN
373
374  flock \${lock} cat > ~/.wgetrc << EOF_WGETRC
375  https_proxy = ${http_proxy}
376  http_proxy = ${http_proxy}
377  use_proxy = on
378EOF_WGETRC
379
380  flock \${lock} cat > ~/.curlrc << EOF_CURLRC
381  proxy = ${PROXY_HOST}:${PROXY_PORT}
382EOF_CURLRC
383fi
384
385# Source our build env
386${BITBAKE_CMD}
387
388if [[ -z "${MACHINE}" ]]; then
389  echo "MACHINE is not configured for ${target}"
390  exit 1
391fi
392
393export MACHINE="${MACHINE}"
394if [[ -z "${DISTRO}" ]]; then
395  echo "DISTRO is not configured for ${target} so will use default"
396  unset DISTRO
397else
398  export DISTRO="${DISTRO}"
399fi
400
401# Custom BitBake config settings
402cat >> conf/local.conf << EOF_CONF
403BB_NUMBER_THREADS = "$(nproc)"
404PARALLEL_MAKE = "-j$(nproc)"
405INHERIT += "rm_work"
406BB_GENERATE_MIRROR_TARBALLS = "1"
407DL_DIR="${ssc_dir}/bitbake_downloads"
408SSTATE_DIR="${ssc_dir}/bitbake_sharedstatecache"
409USER_CLASSES += "buildstats"
410INHERIT_remove = "uninative"
411TMPDIR="${build_dir}"
412EOF_CONF
413
414# Kick off a build
415if [[ -n "${nice_priority}" ]]; then
416    nice -${nice_priority} bitbake ${BITBAKE_OPTS} ${bitbake_target}
417else
418    bitbake ${BITBAKE_OPTS} ${bitbake_target}
419fi
420
421# Copy internal build directory into xtrct_path directory
422if [[ ${xtrct_small_copy_dir} ]]; then
423  mkdir -p ${xtrct_path}/${xtrct_small_copy_dir}
424  timeout ${xtrct_copy_timeout} cp -r ${build_dir}/${xtrct_small_copy_dir}/* ${xtrct_path}/${xtrct_small_copy_dir}
425else
426  timeout ${xtrct_copy_timeout} cp -r ${build_dir}/* ${xtrct_path}
427fi
428
429if [[ 0 -ne $? ]]; then
430  echo "Received a non-zero exit code from timeout"
431  exit 1
432fi
433
434EOF_SCRIPT
435
436chmod a+x ${WORKSPACE}/build.sh
437
438# Give the Docker image a name based on the distro,tag,arch,and target
439img_name=${img_name:-openbmc/${distro}:${img_tag}-${target}-${ARCH}}
440
441# Build the Docker image
442docker build -t ${img_name} - <<< "${Dockerfile}"
443
444# If obmc_dir or ssc_dir are ${HOME} or a subdirectory they will not be mounted
445mount_obmc_dir="-v ""${obmc_dir}"":""${obmc_dir}"" "
446mount_ssc_dir="-v ""${ssc_dir}"":""${ssc_dir}"" "
447mount_workspace_dir="-v ""${WORKSPACE}"":""${WORKSPACE}"" "
448if [[ "${obmc_dir}" = "${HOME}/"* || "${obmc_dir}" = "${HOME}" ]];then
449mount_obmc_dir=""
450fi
451if [[ "${ssc_dir}" = "${HOME}/"* || "${ssc_dir}" = "${HOME}" ]];then
452mount_ssc_dir=""
453fi
454if [[ "${WORKSPACE}" = "${HOME}/"* || "${WORKSPACE}" = "${HOME}" ]];then
455mount_workspace_dir=""
456fi
457
458# Run the Docker container, execute the build.sh script
459docker run \
460--cap-add=sys_admin \
461--cap-add=sys_nice \
462--net=host \
463--rm=true \
464-e WORKSPACE=${WORKSPACE} \
465-w "${HOME}" \
466-v "${HOME}":"${HOME}" \
467${mount_obmc_dir} \
468${mount_ssc_dir} \
469${mount_workspace_dir} \
470--cpus="$num_cpu" \
471-t ${img_name} \
472${WORKSPACE}/build.sh
473
474# To maintain function of resources that used an older path, add a link
475ln -sf ${xtrct_path}/deploy ${WORKSPACE}/deploy
476
477# Timestamp for build
478echo "Build completed, $(date)"
479