xref: /openbmc/openbmc-build-scripts/build-setup.sh (revision d2176ef9704d968858f031e89ace9d1b876d3bf5) 
1#!/bin/bash
2###############################################################################
3#
4# This build script is for running the OpenBMC builds as Docker containers.
5#
6###############################################################################
7#
8# Script Variables:
9#  build_scripts_dir  The path of the openbmc-build-scripts directory.
10#                     Default: The directory containing this script
11#  http_proxy         The HTTP address of the proxy server to connect to.
12#                     Default: "", proxy is not setup if this is not set
13#  WORKSPACE          Path of the workspace directory where some intermediate
14#                     files and the images will be saved to.
15#                     Default: "~/{RandomNumber}"
16#  num_cpu            Number of cpu's to give bitbake, default is total amount
17#                     in system
18#  UBUNTU_MIRROR:     <optional, the URL of a mirror of Ubuntu to override the
19#                     default ones in /etc/apt/sources.list>
20#                     default is empty, and no mirror is used.
21#
22# Docker Image Build Variables:
23#  BITBAKE_OPTS       Set to "-c populate_sdk" or whatever other BitBake options
24#                     you'd like to pass into the build.
25#                     Default: "", no options set
26#  build_dir          Path where the actual BitBake build occurs inside the
27#                     container, path cannot be located on network storage.
28#                     Default: "/tmp/openbmc"
29#  distro             The distro used as the base image for the build image:
30#                     fedora|ubuntu
31#                     Default: "ubuntu"
32#  img_name           The name given to the target build's docker image.
33#                     Default: "openbmc/${distro}:${imgtag}-${target}-${ARCH}"
34#  img_tag            The base docker image distro tag:
35#                     ubuntu: latest|16.04|14.04|trusty|xenial
36#                     fedora: 23|24|25
37#                     Default: "latest"
38#  target             The target we aim to build:
39#                     evb-ast2500|palmetto|qemu|qemux86-64
40#                     romulus|s2600wf|witherspoon|zaius|tiogapass|gsj|mihawk
41#                     witherspoon-tacoma|rainier
42#                     Default: "qemu"
43#  no_tar             Set to true if you do not want the debug tar built
44#                     Default: "false"
45#  nice_priority      Set nice priotity for bitbake command.
46#                     Nice:
47#                       Run with an adjusted niceness, which affects process
48#                       scheduling. Nice values range from -20 (most favorable
49#                       to the process) to 19 (least favorable to the process).
50#                     Default: "", nice is not used if nice_priority is not set
51#
52# Deployment Variables:
53#  obmc_dir           Path of the OpenBMC repo directory used as a reference
54#                     for the build inside the container.
55#                     Default: "${WORKSPACE}/openbmc"
56#  xtrct_small_copy_dir
57#                     Directory within build_dir that should be copied to
58#                     xtrct_path. The directory and all parents up to, but not
59#                     including, build_dir will be copied. For example, if
60#                     build_dir is set to "/tmp/openbmc" and this is set to
61#                     "build/tmp", the directory at xtrct_path will have the
62#                     following directory structure:
63#                     xtrct_path
64#                      | - build
65#                        | - tmp
66#                          ...
67#                     Can also be set to the empty string to copy the entire
68#                     contents of build_dir to xtrct_path.
69#                     Default: "deploy/images".
70#
71###############################################################################
72# Trace bash processing. Set -e so when a step fails, we fail the build
73set -xeo pipefail
74
75# Script Variables:
76build_scripts_dir=${build_scripts_dir:-"$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"}
77http_proxy=${http_proxy:-}
78WORKSPACE=${WORKSPACE:-${HOME}/${RANDOM}${RANDOM}}
79num_cpu=${num_cpu:-$(nproc)}
80UBUNTU_MIRROR=${UBUNTU_MIRROR:-""}
81
82# Docker Image Build Variables:
83build_dir=${build_dir:-/tmp/openbmc}
84distro=${distro:-ubuntu}
85img_tag=${img_tag:-latest}
86target=${target:-qemu}
87no_tar=${no_tar:-false}
88nice_priority=${nice_priority:-}
89
90# Deployment variables
91obmc_dir=${obmc_dir:-${WORKSPACE}/openbmc}
92ssc_dir=${HOME}
93xtrct_small_copy_dir=${xtrct_small_copy_dir:-deploy/images}
94xtrct_path="${obmc_dir}/build/tmp"
95xtrct_copy_timeout="300"
96
97bitbake_target="obmc-phosphor-image"
98PROXY=""
99
100MIRROR=""
101if [[ -n "${UBUNTU_MIRROR}" ]]; then
102    MIRROR="RUN echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME) main restricted universe multiverse\" > /etc/apt/sources.list && \
103        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-updates main restricted universe multiverse\" >> /etc/apt/sources.list && \
104        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-security main restricted universe multiverse\" >> /etc/apt/sources.list && \
105        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-proposed main restricted universe multiverse\" >> /etc/apt/sources.list && \
106        echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-backports main restricted universe multiverse\" >> /etc/apt/sources.list"
107fi
108
109# Determine the architecture
110ARCH=$(uname -m)
111
112# Determine the prefix of the Dockerfile's base image
113case ${ARCH} in
114  "ppc64le")
115    DOCKER_BASE="ppc64le/"
116    ;;
117  "x86_64")
118    DOCKER_BASE=""
119    ;;
120  *)
121    echo "Unsupported system architecture(${ARCH}) found for docker image"
122    exit 1
123esac
124
125# Timestamp for job
126echo "Build started, $(date)"
127
128# If the obmc_dir directory doesn't exist clone it in
129if [ ! -d ${obmc_dir} ]; then
130  echo "Clone in openbmc master to ${obmc_dir}"
131  git clone https://github.com/openbmc/openbmc ${obmc_dir}
132fi
133
134# Make and chown the xtrct_path directory to avoid permission errors
135if [ ! -d ${xtrct_path} ]; then
136  mkdir -p ${xtrct_path}
137fi
138chown ${UID}:${GROUPS} ${xtrct_path}
139
140# Work out what build target we should be running and set BitBake command
141MACHINE=""
142case ${target} in
143  palmetto)
144    LAYER_DIR="meta-ibm/meta-palmetto"
145    MACHINE="palmetto"
146    DISTRO="openbmc-openpower"
147    ;;
148  swift)
149    LAYER_DIR="meta-ibm"
150    MACHINE="swift"
151    DISTRO="openbmc-witherspoon"
152    ;;
153  mihawk)
154    LAYER_DIR="meta-ibm"
155    MACHINE="mihawk"
156    DISTRO="openbmc-witherspoon"
157    ;;
158  witherspoon)
159    LAYER_DIR="meta-ibm"
160    MACHINE="witherspoon"
161    DISTRO="openbmc-witherspoon"
162    ;;
163  witherspoon-128)
164    LAYER_DIR="meta-ibm"
165    MACHINE="witherspoon-128"
166    DISTRO="openbmc-witherspoon"
167    ;;
168  witherspoon-tacoma)
169    LAYER_DIR="meta-ibm"
170    MACHINE="witherspoon-tacoma"
171    DISTRO="openbmc-openpower"
172    ;;
173  rainier)
174    LAYER_DIR="meta-ibm"
175    MACHINE="rainier"
176    DISTRO="openbmc-openpower"
177    ;;
178  evb-ast2500)
179    LAYER_DIR="meta-evb/meta-evb-aspeed/meta-evb-ast2500"
180    MACHINE="evb-ast2500"
181    DISTRO="openbmc-phosphor"
182    ;;
183  s2600wf)
184    LAYER_DIR="meta-intel/meta-s2600wf"
185    MACHINE="s2600wf"
186    DISTRO="openbmc-phosphor"
187    ;;
188  zaius)
189    LAYER_DIR="meta-ingrasys/meta-zaius"
190    MACHINE="zaius"
191    DISTRO="openbmc-openpower"
192    ;;
193  romulus)
194    LAYER_DIR="meta-ibm/meta-romulus"
195    MACHINE="romulus"
196    DISTRO="openbmc-openpower"
197    ;;
198  tiogapass)
199    LAYER_DIR="meta-facebook/meta-tiogapass"
200    MACHINE="tiogapass"
201    DISTRO="openbmc-phosphor"
202    ;;
203  gsj)
204    LAYER_DIR="meta-quanta/meta-gsj"
205    MACHINE="gsj"
206    # Use default DISTRO from layer
207    ;;
208  *)
209    exit 1
210    ;;
211esac
212
213BITBAKE_CMD="TEMPLATECONF=${LAYER_DIR}/conf source oe-init-build-env"
214
215# Configure Docker build
216if [[ "${distro}" == fedora ]];then
217
218  if [[ -n "${http_proxy}" ]]; then
219    PROXY="RUN echo \"proxy=${http_proxy}\" >> /etc/dnf/dnf.conf"
220  fi
221
222  Dockerfile=$(cat << EOF
223  FROM ${DOCKER_BASE}${distro}:${img_tag}
224
225  ${PROXY}
226
227  # Set the locale
228  RUN locale-gen en_US.UTF-8
229  ENV LANG en_US.UTF-8
230  ENV LANGUAGE en_US:en
231  ENV LC_ALL en_US.UTF-8
232
233  RUN dnf --refresh install -y \
234      bzip2 \
235      chrpath \
236      cpio \
237      diffstat \
238      findutils \
239      gcc \
240      gcc-c++ \
241      git \
242      make \
243      patch \
244      perl-bignum \
245      perl-Data-Dumper \
246      perl-Thread-Queue \
247      python-devel \
248      python3-devel \
249      SDL-devel \
250      socat \
251      subversion \
252      tar \
253      texinfo \
254      wget \
255      which \
256      iputils-ping
257
258  RUN grep -q ${GROUPS} /etc/group || groupadd -g ${GROUPS} ${USER}
259  RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS} ${USER}
260
261  USER ${USER}
262  ENV HOME ${HOME}
263  RUN /bin/bash
264EOF
265)
266
267elif [[ "${distro}" == ubuntu ]]; then
268
269  if [[ -n "${http_proxy}" ]]; then
270    PROXY="RUN echo \"Acquire::http::Proxy \\"\"${http_proxy}/\\"\";\" > /etc/apt/apt.conf.d/000apt-cacher-ng-proxy"
271  fi
272
273  Dockerfile=$(cat << EOF
274  FROM ${DOCKER_BASE}${distro}:${img_tag}
275
276  ${PROXY}
277  ${MIRROR}
278
279  ENV DEBIAN_FRONTEND noninteractive
280
281  RUN apt-get update && apt-get install -yy \
282      build-essential \
283      chrpath \
284      debianutils \
285      diffstat \
286      gawk \
287      git \
288      libdata-dumper-simple-perl \
289      libsdl1.2-dev \
290      libthread-queue-any-perl \
291      locales \
292      python \
293      python3 \
294      socat \
295      subversion \
296      texinfo \
297      cpio \
298      wget \
299      iputils-ping
300
301  # Set the locale
302  RUN locale-gen en_US.UTF-8
303  ENV LANG en_US.UTF-8
304  ENV LANGUAGE en_US:en
305  ENV LC_ALL en_US.UTF-8
306
307  RUN grep -q ${GROUPS} /etc/group || groupadd -g ${GROUPS} ${USER}
308  RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS} ${USER}
309
310  USER ${USER}
311  ENV HOME ${HOME}
312  RUN /bin/bash
313EOF
314)
315fi
316
317# Create the Docker run script
318export PROXY_HOST=${http_proxy/#http*:\/\/}
319export PROXY_HOST=${PROXY_HOST/%:[0-9]*}
320export PROXY_PORT=${http_proxy/#http*:\/\/*:}
321
322mkdir -p ${WORKSPACE}
323
324# Determine command for bitbake image build
325if [ $no_tar = "false" ]; then
326    bitbake_target="${bitbake_target} obmc-phosphor-debug-tarball"
327fi
328
329cat > "${WORKSPACE}"/build.sh << EOF_SCRIPT
330#!/bin/bash
331
332set -xeo pipefail
333
334# Go into the OpenBMC directory, the build will handle changing directories
335cd ${obmc_dir}
336
337# Set up proxies
338export ftp_proxy=${http_proxy}
339export http_proxy=${http_proxy}
340export https_proxy=${http_proxy}
341
342mkdir -p ${WORKSPACE}/bin
343
344# Configure proxies for BitBake
345if [[ -n "${http_proxy}" ]]; then
346
347  cat > ${WORKSPACE}/bin/git-proxy << \EOF_GIT
348  #!/bin/bash
349  # \$1 = hostname, \$2 = port
350  PROXY=${PROXY_HOST}
351  PROXY_PORT=${PROXY_PORT}
352  exec socat STDIO PROXY:\${PROXY}:\${1}:\${2},proxyport=\${PROXY_PORT}
353EOF_GIT
354
355  chmod a+x ${WORKSPACE}/bin/git-proxy
356  export PATH=${WORKSPACE}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH}
357
358  lock=${HOME}/build-setup.lock
359  flock \${lock} git config --global core.gitProxy ${WORKSPACE}/bin/git-proxy
360  flock \${lock} git config --global http.proxy ${http_proxy}
361
362  flock \${lock} mkdir -p ~/.subversion
363  flock \${lock} cat > ~/.subversion/servers << EOF_SVN
364  [global]
365  http-proxy-host = ${PROXY_HOST}
366  http-proxy-port = ${PROXY_PORT}
367EOF_SVN
368
369  flock \${lock} cat > ~/.wgetrc << EOF_WGETRC
370  https_proxy = ${http_proxy}
371  http_proxy = ${http_proxy}
372  use_proxy = on
373EOF_WGETRC
374
375  flock \${lock} cat > ~/.curlrc << EOF_CURLRC
376  proxy = ${PROXY_HOST}:${PROXY_PORT}
377EOF_CURLRC
378fi
379
380# Source our build env
381${BITBAKE_CMD}
382
383if [[ -z "${MACHINE}" ]]; then
384  echo "MACHINE is not configured for ${target}"
385  exit 1
386fi
387
388export MACHINE="${MACHINE}"
389if [[ -z "${DISTRO}" ]]; then
390  echo "DISTRO is not configured for ${target} so will use default"
391  unset DISTRO
392else
393  export DISTRO="${DISTRO}"
394fi
395
396# Custom BitBake config settings
397cat >> conf/local.conf << EOF_CONF
398BB_NUMBER_THREADS = "$(nproc)"
399PARALLEL_MAKE = "-j$(nproc)"
400INHERIT += "rm_work"
401BB_GENERATE_MIRROR_TARBALLS = "1"
402DL_DIR="${ssc_dir}/bitbake_downloads"
403SSTATE_DIR="${ssc_dir}/bitbake_sharedstatecache"
404USER_CLASSES += "buildstats"
405INHERIT_remove = "uninative"
406TMPDIR="${build_dir}"
407EOF_CONF
408
409# Kick off a build
410if [[ -n "${nice_priority}" ]]; then
411    nice -${nice_priority} bitbake ${BITBAKE_OPTS} ${bitbake_target}
412else
413    bitbake ${BITBAKE_OPTS} ${bitbake_target}
414fi
415
416# Copy internal build directory into xtrct_path directory
417if [[ ${xtrct_small_copy_dir} ]]; then
418  mkdir -p ${xtrct_path}/${xtrct_small_copy_dir}
419  timeout ${xtrct_copy_timeout} cp -r ${build_dir}/${xtrct_small_copy_dir}/* ${xtrct_path}/${xtrct_small_copy_dir}
420else
421  timeout ${xtrct_copy_timeout} cp -r ${build_dir}/* ${xtrct_path}
422fi
423
424if [[ 0 -ne $? ]]; then
425  echo "Received a non-zero exit code from timeout"
426  exit 1
427fi
428
429EOF_SCRIPT
430
431chmod a+x ${WORKSPACE}/build.sh
432
433# Give the Docker image a name based on the distro,tag,arch,and target
434img_name=${img_name:-openbmc/${distro}:${img_tag}-${target}-${ARCH}}
435
436# Build the Docker image
437docker build -t ${img_name} - <<< "${Dockerfile}"
438
439# If obmc_dir or ssc_dir are ${HOME} or a subdirectory they will not be mounted
440mount_obmc_dir="-v ""${obmc_dir}"":""${obmc_dir}"" "
441mount_ssc_dir="-v ""${ssc_dir}"":""${ssc_dir}"" "
442mount_workspace_dir="-v ""${WORKSPACE}"":""${WORKSPACE}"" "
443if [[ "${obmc_dir}" = "${HOME}/"* || "${obmc_dir}" = "${HOME}" ]];then
444mount_obmc_dir=""
445fi
446if [[ "${ssc_dir}" = "${HOME}/"* || "${ssc_dir}" = "${HOME}" ]];then
447mount_ssc_dir=""
448fi
449if [[ "${WORKSPACE}" = "${HOME}/"* || "${WORKSPACE}" = "${HOME}" ]];then
450mount_workspace_dir=""
451fi
452
453# Run the Docker container, execute the build.sh script
454docker run \
455--cap-add=sys_admin \
456--cap-add=sys_nice \
457--net=host \
458--rm=true \
459-e WORKSPACE=${WORKSPACE} \
460-w "${HOME}" \
461-v "${HOME}":"${HOME}" \
462${mount_obmc_dir} \
463${mount_ssc_dir} \
464${mount_workspace_dir} \
465--cpus="$num_cpu" \
466-t ${img_name} \
467${WORKSPACE}/build.sh
468
469# To maintain function of resources that used an older path, add a link
470ln -sf ${xtrct_path}/deploy ${WORKSPACE}/deploy
471
472# Timestamp for build
473echo "Build completed, $(date)"
474