xref: /openbmc/openbmc-build-scripts/build-setup.sh (revision d374874d201a020e93c1872a42a55ea5f8106d7f) 
1#!/bin/bash
2###############################################################################
3#
4# This build script is for running the OpenBMC builds as Docker containers.
5#
6###############################################################################
7#
8# Script Variables:
9#  build_scripts_dir  The path of the openbmc-build-scripts directory.
10#                     Default: The directory containing this script
11#  http_proxy         The HTTP address of the proxy server to connect to.
12#                     Default: "", proxy is not setup if this is not set
13#  WORKSPACE          Path of the workspace directory where some intermediate
14#                     files and the images will be saved to.
15#                     Default: "~/{RandomNumber}"
16#  num_cpu            Number of cpu's to give bitbake, default is total amount
17#                     in system
18#
19# Docker Image Build Variables:
20#  BITBAKE_OPTS       Set to "-c populate_sdk" or whatever other BitBake options
21#                     you'd like to pass into the build.
22#                     Default: "", no options set
23#  build_dir          Path where the actual BitBake build occurs inside the
24#                     container, path cannot be located on network storage.
25#                     Default: "/tmp/openbmc"
26#  distro             The distro used as the base image for the build image:
27#                     fedora|ubuntu
28#                     Default: "ubuntu"
29#  img_name           The name given to the target build's docker image.
30#                     Default: "openbmc/${distro}:${imgtag}-${target}-${ARCH}"
31#  img_tag            The base docker image distro tag:
32#                     ubuntu: latest|16.04|14.04|trusty|xenial
33#                     fedora: 23|24|25
34#                     Default: "latest"
35#  target             The target we aim to build:
36#                     evb-ast2500|palmetto|qemu|qemux86-64
37#                     romulus|s2600wf|witherspoon|zaius|tiogapass|gsj|mihawk
38#                     witherspoon-tacoma|rainier
39#                     Default: "qemu"
40#  no_tar             Set to true if you do not want the debug tar built
41#                     Default: "false"
42#  nice_priority      Set nice priotity for bitbake command.
43#                     Nice:
44#                       Run with an adjusted niceness, which affects process
45#                       scheduling. Nice values range from -20 (most favorable
46#                       to the process) to 19 (least favorable to the process).
47#                     Default: "", nice is not used if nice_priority is not set
48#
49# Deployment Variables:
50#  obmc_dir           Path of the OpenBMC repo directory used as a reference
51#                     for the build inside the container.
52#                     Default: "${WORKSPACE}/openbmc"
53#  xtrct_small_copy_dir
54#                     Directory within build_dir that should be copied to
55#                     xtrct_path. The directory and all parents up to, but not
56#                     including, build_dir will be copied. For example, if
57#                     build_dir is set to "/tmp/openbmc" and this is set to
58#                     "build/tmp", the directory at xtrct_path will have the
59#                     following directory structure:
60#                     xtrct_path
61#                      | - build
62#                        | - tmp
63#                          ...
64#                     Can also be set to the empty string to copy the entire
65#                     contents of build_dir to xtrct_path.
66#                     Default: "deploy/images".
67#
68###############################################################################
69# Trace bash processing. Set -e so when a step fails, we fail the build
70set -xeo pipefail
71
72# Script Variables:
73build_scripts_dir=${build_scripts_dir:-"$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"}
74http_proxy=${http_proxy:-}
75WORKSPACE=${WORKSPACE:-${HOME}/${RANDOM}${RANDOM}}
76num_cpu=${num_cpu:-$(nproc)}
77
78# Docker Image Build Variables:
79build_dir=${build_dir:-/tmp/openbmc}
80distro=${distro:-ubuntu}
81img_tag=${img_tag:-latest}
82target=${target:-qemu}
83no_tar=${no_tar:-false}
84nice_priority=${nice_priority:-}
85
86# Deployment variables
87obmc_dir=${obmc_dir:-${WORKSPACE}/openbmc}
88ssc_dir=${HOME}
89xtrct_small_copy_dir=${xtrct_small_copy_dir:-deploy/images}
90xtrct_path="${obmc_dir}/build/tmp"
91xtrct_copy_timeout="300"
92
93bitbake_target="obmc-phosphor-image"
94PROXY=""
95
96# Determine the architecture
97ARCH=$(uname -m)
98
99# Determine the prefix of the Dockerfile's base image
100case ${ARCH} in
101  "ppc64le")
102    DOCKER_BASE="ppc64le/"
103    ;;
104  "x86_64")
105    DOCKER_BASE=""
106    ;;
107  *)
108    echo "Unsupported system architecture(${ARCH}) found for docker image"
109    exit 1
110esac
111
112# Timestamp for job
113echo "Build started, $(date)"
114
115# If the obmc_dir directory doesn't exist clone it in
116if [ ! -d ${obmc_dir} ]; then
117  echo "Clone in openbmc master to ${obmc_dir}"
118  git clone https://github.com/openbmc/openbmc ${obmc_dir}
119fi
120
121# Make and chown the xtrct_path directory to avoid permission errors
122if [ ! -d ${xtrct_path} ]; then
123  mkdir -p ${xtrct_path}
124fi
125chown ${UID}:${GROUPS} ${xtrct_path}
126
127# Work out what build target we should be running and set BitBake command
128MACHINE=""
129case ${target} in
130  palmetto)
131    LAYER_DIR="meta-ibm/meta-palmetto"
132    MACHINE="palmetto"
133    DISTRO="openbmc-openpower"
134    ;;
135  swift)
136    LAYER_DIR="meta-ibm"
137    MACHINE="swift"
138    DISTRO="openbmc-witherspoon"
139    ;;
140  mihawk)
141    LAYER_DIR="meta-ibm"
142    MACHINE="mihawk"
143    DISTRO="openbmc-witherspoon"
144    ;;
145  witherspoon)
146    LAYER_DIR="meta-ibm"
147    MACHINE="witherspoon"
148    DISTRO="openbmc-witherspoon"
149    ;;
150  witherspoon-128)
151    LAYER_DIR="meta-ibm"
152    MACHINE="witherspoon-128"
153    DISTRO="openbmc-witherspoon"
154    ;;
155  witherspoon-tacoma)
156    LAYER_DIR="meta-ibm"
157    MACHINE="witherspoon-tacoma"
158    DISTRO="openbmc-openpower"
159    ;;
160  rainier)
161    LAYER_DIR="meta-ibm"
162    MACHINE="rainier"
163    DISTRO="openbmc-openpower"
164    ;;
165  evb-ast2500)
166    LAYER_DIR="meta-evb/meta-evb-aspeed/meta-evb-ast2500"
167    MACHINE="evb-ast2500"
168    DISTRO="openbmc-phosphor"
169    ;;
170  s2600wf)
171    LAYER_DIR="meta-intel/meta-s2600wf"
172    MACHINE="s2600wf"
173    DISTRO="openbmc-phosphor"
174    ;;
175  zaius)
176    LAYER_DIR="meta-ingrasys/meta-zaius"
177    MACHINE="zaius"
178    DISTRO="openbmc-openpower"
179    ;;
180  romulus)
181    LAYER_DIR="meta-ibm/meta-romulus"
182    MACHINE="romulus"
183    DISTRO="openbmc-openpower"
184    ;;
185  tiogapass)
186    LAYER_DIR="meta-facebook/meta-tiogapass"
187    MACHINE="tiogapass"
188    DISTRO="openbmc-phosphor"
189    ;;
190  gsj)
191    LAYER_DIR="meta-quanta/meta-gsj"
192    MACHINE="gsj"
193    # Use default DISTRO from layer
194    ;;
195  *)
196    exit 1
197    ;;
198esac
199
200BITBAKE_CMD="TEMPLATECONF=${LAYER_DIR}/conf source oe-init-build-env"
201
202# Configure Docker build
203if [[ "${distro}" == fedora ]];then
204
205  if [[ -n "${http_proxy}" ]]; then
206    PROXY="RUN echo \"proxy=${http_proxy}\" >> /etc/dnf/dnf.conf"
207  fi
208
209  Dockerfile=$(cat << EOF
210  FROM ${DOCKER_BASE}${distro}:${img_tag}
211
212  ${PROXY}
213
214  # Set the locale
215  RUN locale-gen en_US.UTF-8
216  ENV LANG en_US.UTF-8
217  ENV LANGUAGE en_US:en
218  ENV LC_ALL en_US.UTF-8
219
220  RUN dnf --refresh install -y \
221      bzip2 \
222      chrpath \
223      cpio \
224      diffstat \
225      findutils \
226      gcc \
227      gcc-c++ \
228      git \
229      make \
230      patch \
231      perl-bignum \
232      perl-Data-Dumper \
233      perl-Thread-Queue \
234      python-devel \
235      python3-devel \
236      SDL-devel \
237      socat \
238      subversion \
239      tar \
240      texinfo \
241      wget \
242      which \
243      iputils-ping
244
245  RUN grep -q ${GROUPS} /etc/group || groupadd -g ${GROUPS} ${USER}
246  RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS} ${USER}
247
248  USER ${USER}
249  ENV HOME ${HOME}
250  RUN /bin/bash
251EOF
252)
253
254elif [[ "${distro}" == ubuntu ]]; then
255
256  if [[ -n "${http_proxy}" ]]; then
257    PROXY="RUN echo \"Acquire::http::Proxy \\"\"${http_proxy}/\\"\";\" > /etc/apt/apt.conf.d/000apt-cacher-ng-proxy"
258  fi
259
260  Dockerfile=$(cat << EOF
261  FROM ${DOCKER_BASE}${distro}:${img_tag}
262
263  ${PROXY}
264
265  ENV DEBIAN_FRONTEND noninteractive
266
267  RUN apt-get update && apt-get install -yy \
268      build-essential \
269      chrpath \
270      debianutils \
271      diffstat \
272      gawk \
273      git \
274      libdata-dumper-simple-perl \
275      libsdl1.2-dev \
276      libthread-queue-any-perl \
277      locales \
278      python \
279      python3 \
280      socat \
281      subversion \
282      texinfo \
283      cpio \
284      wget \
285      iputils-ping
286
287  # Set the locale
288  RUN locale-gen en_US.UTF-8
289  ENV LANG en_US.UTF-8
290  ENV LANGUAGE en_US:en
291  ENV LC_ALL en_US.UTF-8
292
293  RUN grep -q ${GROUPS} /etc/group || groupadd -g ${GROUPS} ${USER}
294  RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS} ${USER}
295
296  USER ${USER}
297  ENV HOME ${HOME}
298  RUN /bin/bash
299EOF
300)
301fi
302
303# Create the Docker run script
304export PROXY_HOST=${http_proxy/#http*:\/\/}
305export PROXY_HOST=${PROXY_HOST/%:[0-9]*}
306export PROXY_PORT=${http_proxy/#http*:\/\/*:}
307
308mkdir -p ${WORKSPACE}
309
310# Determine command for bitbake image build
311if [ $no_tar = "false" ]; then
312    bitbake_target="${bitbake_target} obmc-phosphor-debug-tarball"
313fi
314
315cat > "${WORKSPACE}"/build.sh << EOF_SCRIPT
316#!/bin/bash
317
318set -xeo pipefail
319
320# Go into the OpenBMC directory, the build will handle changing directories
321cd ${obmc_dir}
322
323# Set up proxies
324export ftp_proxy=${http_proxy}
325export http_proxy=${http_proxy}
326export https_proxy=${http_proxy}
327
328mkdir -p ${WORKSPACE}/bin
329
330# Configure proxies for BitBake
331if [[ -n "${http_proxy}" ]]; then
332
333  cat > ${WORKSPACE}/bin/git-proxy << \EOF_GIT
334  #!/bin/bash
335  # \$1 = hostname, \$2 = port
336  PROXY=${PROXY_HOST}
337  PROXY_PORT=${PROXY_PORT}
338  exec socat STDIO PROXY:\${PROXY}:\${1}:\${2},proxyport=\${PROXY_PORT}
339EOF_GIT
340
341  chmod a+x ${WORKSPACE}/bin/git-proxy
342  export PATH=${WORKSPACE}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH}
343  git config core.gitProxy git-proxy
344
345  mkdir -p ~/.subversion
346
347  cat > ~/.subversion/servers << EOF_SVN
348  [global]
349  http-proxy-host = ${PROXY_HOST}
350  http-proxy-port = ${PROXY_PORT}
351EOF_SVN
352
353  cat > ~/.wgetrc << EOF_WGETRC
354  https_proxy = ${http_proxy}
355  http_proxy = ${http_proxy}
356  use_proxy = on
357EOF_WGETRC
358
359  cat > ~/.curlrc << EOF_CURLRC
360  proxy = ${PROXY_HOST}:${PROXY_PORT}
361EOF_CURLRC
362fi
363
364# Source our build env
365${BITBAKE_CMD}
366
367if [[ -z "${MACHINE}" ]]; then
368  echo "MACHINE is not configured for ${target}"
369  exit 1
370fi
371
372export MACHINE="${MACHINE}"
373if [[ -z "${DISTRO}" ]]; then
374  echo "DISTRO is not configured for ${target} so will use default"
375  unset DISTRO
376else
377  export DISTRO="${DISTRO}"
378fi
379
380# Custom BitBake config settings
381cat >> conf/local.conf << EOF_CONF
382BB_NUMBER_THREADS = "$(nproc)"
383PARALLEL_MAKE = "-j$(nproc)"
384INHERIT += "rm_work"
385BB_GENERATE_MIRROR_TARBALLS = "1"
386DL_DIR="${ssc_dir}/bitbake_downloads"
387SSTATE_DIR="${ssc_dir}/bitbake_sharedstatecache"
388USER_CLASSES += "buildstats"
389INHERIT_remove = "uninative"
390TMPDIR="${build_dir}"
391EOF_CONF
392
393# Kick off a build
394if [[ -n "${nice_priority}" ]]; then
395    nice -${nice_priority} bitbake ${BITBAKE_OPTS} ${bitbake_target}
396else
397    bitbake ${BITBAKE_OPTS} ${bitbake_target}
398fi
399
400# Copy internal build directory into xtrct_path directory
401if [[ ${xtrct_small_copy_dir} ]]; then
402  mkdir -p ${xtrct_path}/${xtrct_small_copy_dir}
403  timeout ${xtrct_copy_timeout} cp -r ${build_dir}/${xtrct_small_copy_dir}/* ${xtrct_path}/${xtrct_small_copy_dir}
404else
405  timeout ${xtrct_copy_timeout} cp -r ${build_dir}/* ${xtrct_path}
406fi
407
408if [[ 0 -ne $? ]]; then
409  echo "Received a non-zero exit code from timeout"
410  exit 1
411fi
412
413EOF_SCRIPT
414
415chmod a+x ${WORKSPACE}/build.sh
416
417# Give the Docker image a name based on the distro,tag,arch,and target
418img_name=${img_name:-openbmc/${distro}:${img_tag}-${target}-${ARCH}}
419
420# Build the Docker image
421docker build -t ${img_name} - <<< "${Dockerfile}"
422
423# If obmc_dir or ssc_dir are ${HOME} or a subdirectory they will not be mounted
424mount_obmc_dir="-v ""${obmc_dir}"":""${obmc_dir}"" "
425mount_ssc_dir="-v ""${ssc_dir}"":""${ssc_dir}"" "
426mount_workspace_dir="-v ""${WORKSPACE}"":""${WORKSPACE}"" "
427if [[ "${obmc_dir}" = "${HOME}/"* || "${obmc_dir}" = "${HOME}" ]];then
428mount_obmc_dir=""
429fi
430if [[ "${ssc_dir}" = "${HOME}/"* || "${ssc_dir}" = "${HOME}" ]];then
431mount_ssc_dir=""
432fi
433if [[ "${WORKSPACE}" = "${HOME}/"* || "${WORKSPACE}" = "${HOME}" ]];then
434mount_workspace_dir=""
435fi
436
437# Run the Docker container, execute the build.sh script
438docker run \
439--cap-add=sys_admin \
440--cap-add=sys_nice \
441--net=host \
442--rm=true \
443-e WORKSPACE=${WORKSPACE} \
444-w "${HOME}" \
445-v "${HOME}":"${HOME}" \
446${mount_obmc_dir} \
447${mount_ssc_dir} \
448${mount_workspace_dir} \
449--cpus="$num_cpu" \
450-t ${img_name} \
451${WORKSPACE}/build.sh
452
453# To maintain function of resources that used an older path, add a link
454ln -sf ${xtrct_path}/deploy ${WORKSPACE}/deploy
455
456# Timestamp for build
457echo "Build completed, $(date)"
458