xref: /openbmc/openbmc-build-scripts/build-setup.sh (revision a4d6d3d26ab0c1ecbda487c762277542fb8e26e4) 
1#!/bin/bash
2###############################################################################
3#
4# This build script is for running the OpenBMC builds as Docker containers.
5#
6###############################################################################
7#
8# Script Variables:
9#  build_scripts_dir  The path of the openbmc-build-scripts directory.
10#                     Default: The directory containing this script
11#  http_proxy         The HTTP address of the proxy server to connect to.
12#                     Default: "", proxy is not setup if this is not set
13#  WORKSPACE          Path of the workspace directory where some intermediate
14#                     files and the images will be saved to.
15#                     Default: "~/{RandomNumber}"
16#  num_cpu            Number of cpu's to give bitbake, default is total amount
17#                     in system
18#
19# Docker Image Build Variables:
20#  BITBAKE_OPTS       Set to "-c populate_sdk" or whatever other BitBake options
21#                     you'd like to pass into the build.
22#                     Default: "", no options set
23#  build_dir          Path where the actual BitBake build occurs inside the
24#                     container, path cannot be located on network storage.
25#                     Default: "/tmp/openbmc"
26#  distro             The distro used as the base image for the build image:
27#                     fedora|ubuntu
28#                     Default: "ubuntu"
29#  img_name           The name given to the target build's docker image.
30#                     Default: "openbmc/${distro}:${imgtag}-${target}-${ARCH}"
31#  img_tag            The base docker image distro tag:
32#                     ubuntu: latest|16.04|14.04|trusty|xenial
33#                     fedora: 23|24|25
34#                     Default: "latest"
35#  target             The target we aim to build:
36#                     evb-ast2500|palmetto|qemu|qemux86-64
37#                     romulus|s2600wf|witherspoon|zaius|tiogapass|gsj|mihawk
38#                     Default: "qemu"
39#  no_tar             Set to true if you do not want the debug tar built
40#                     Default: "false"
41#  nice_priority      Set nice priotity for bitbake command.
42#                     Nice:
43#                       Run with an adjusted niceness, which affects process
44#                       scheduling. Nice values range from -20 (most favorable
45#                       to the process) to 19 (least favorable to the process).
46#                     Default: "", nice is not used if nice_priority is not set
47#
48# Deployment Variables:
49#  obmc_dir           Path of the OpenBMC repo directory used as a reference
50#                     for the build inside the container.
51#                     Default: "${WORKSPACE}/openbmc"
52#  xtrct_small_copy_dir
53#                     Directory within build_dir that should be copied to
54#                     xtrct_path. The directory and all parents up to, but not
55#                     including, build_dir will be copied. For example, if
56#                     build_dir is set to "/tmp/openbmc" and this is set to
57#                     "build/tmp", the directory at xtrct_path will have the
58#                     following directory structure:
59#                     xtrct_path
60#                      | - build
61#                        | - tmp
62#                          ...
63#                     Can also be set to the empty string to copy the entire
64#                     contents of build_dir to xtrct_path.
65#                     Default: "deploy/images".
66#
67###############################################################################
68# Trace bash processing. Set -e so when a step fails, we fail the build
69set -xeo pipefail
70
71# Script Variables:
72build_scripts_dir=${build_scripts_dir:-"$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"}
73http_proxy=${http_proxy:-}
74WORKSPACE=${WORKSPACE:-${HOME}/${RANDOM}${RANDOM}}
75num_cpu=${num_cpu:-$(nproc)}
76
77# Docker Image Build Variables:
78build_dir=${build_dir:-/tmp/openbmc}
79distro=${distro:-ubuntu}
80img_tag=${img_tag:-latest}
81target=${target:-qemu}
82no_tar=${no_tar:-false}
83nice_priority=${nice_priority:-}
84
85# Deployment variables
86obmc_dir=${obmc_dir:-${WORKSPACE}/openbmc}
87ssc_dir=${HOME}
88xtrct_small_copy_dir=${xtrct_small_copy_dir:-deploy/images}
89xtrct_path="${obmc_dir}/build/tmp"
90xtrct_copy_timeout="300"
91
92PROXY=""
93
94# Determine the architecture
95ARCH=$(uname -m)
96
97# Determine the prefix of the Dockerfile's base image
98case ${ARCH} in
99  "ppc64le")
100    DOCKER_BASE="ppc64le/"
101    ;;
102  "x86_64")
103    DOCKER_BASE=""
104    ;;
105  *)
106    echo "Unsupported system architecture(${ARCH}) found for docker image"
107    exit 1
108esac
109
110# Timestamp for job
111echo "Build started, $(date)"
112
113# If the obmc_dir directory doesn't exist clone it in
114if [ ! -d ${obmc_dir} ]; then
115  echo "Clone in openbmc master to ${obmc_dir}"
116  git clone https://github.com/openbmc/openbmc ${obmc_dir}
117fi
118
119# Make and chown the xtrct_path directory to avoid permission errors
120if [ ! -d ${xtrct_path} ]; then
121  mkdir -p ${xtrct_path}
122fi
123chown ${UID}:${GROUPS} ${xtrct_path}
124
125# Work out what build target we should be running and set BitBake command
126MACHINE=""
127case ${target} in
128  palmetto)
129    LAYER_DIR="meta-ibm/meta-palmetto"
130    ;;
131  swift)
132    LAYER_DIR="meta-ibm/meta-witherspoon"
133    MACHINE="swift"
134    ;;
135  mihawk)
136    LAYER_DIR="meta-ibm/meta-witherspoon"
137    MACHINE="mihawk"
138    ;;
139  witherspoon)
140    LAYER_DIR="meta-ibm/meta-witherspoon"
141    ;;
142  witherspoon-128)
143    LAYER_DIR="meta-ibm/meta-witherspoon"
144    MACHINE="witherspoon-128"
145    ;;
146  evb-ast2500)
147    LAYER_DIR="meta-evb/meta-evb-aspeed/meta-evb-ast2500"
148    ;;
149  s2600wf)
150    LAYER_DIR="meta-intel/meta-s2600wf"
151    ;;
152  zaius)
153    LAYER_DIR="meta-ingrasys/meta-zaius"
154    ;;
155  romulus)
156    LAYER_DIR="meta-ibm/meta-romulus"
157    ;;
158  qemu)
159    LAYER_DIR="meta-phosphor"
160    # MACHINE defaults to `qemuarm` in this layer, no change necessary
161    ;;
162  qemux86-64)
163    LAYER_DIR="meta-phosphor"
164    # MACHINE defaults to `qemuarm` in this layer, change to `qemux86-64`
165    MACHINE="qemux86-64"
166    ;;
167  tiogapass)
168    LAYER_DIR="meta-facebook/meta-tiogapass"
169    ;;
170  gsj)
171    LAYER_DIR="meta-quanta/meta-gsj"
172    ;;
173  *)
174    exit 1
175    ;;
176esac
177
178BITBAKE_CMD="TEMPLATECONF=${LAYER_DIR}/conf source oe-init-build-env"
179
180# Configure Docker build
181if [[ "${distro}" == fedora ]];then
182
183  if [[ -n "${http_proxy}" ]]; then
184    PROXY="RUN echo \"proxy=${http_proxy}\" >> /etc/dnf/dnf.conf"
185  fi
186
187  Dockerfile=$(cat << EOF
188  FROM ${DOCKER_BASE}${distro}:${img_tag}
189
190  ${PROXY}
191
192  # Set the locale
193  RUN locale-gen en_US.UTF-8
194  ENV LANG en_US.UTF-8
195  ENV LANGUAGE en_US:en
196  ENV LC_ALL en_US.UTF-8
197
198  RUN dnf --refresh install -y \
199      bzip2 \
200      chrpath \
201      cpio \
202      diffstat \
203      findutils \
204      gcc \
205      gcc-c++ \
206      git \
207      make \
208      patch \
209      perl-bignum \
210      perl-Data-Dumper \
211      perl-Thread-Queue \
212      python-devel \
213      python3-devel \
214      SDL-devel \
215      socat \
216      subversion \
217      tar \
218      texinfo \
219      wget \
220      which \
221      iputils-ping
222
223  RUN grep -q ${GROUPS} /etc/group || groupadd -g ${GROUPS} ${USER}
224  RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS} ${USER}
225
226  USER ${USER}
227  ENV HOME ${HOME}
228  RUN /bin/bash
229EOF
230)
231
232elif [[ "${distro}" == ubuntu ]]; then
233
234  if [[ -n "${http_proxy}" ]]; then
235    PROXY="RUN echo \"Acquire::http::Proxy \\"\"${http_proxy}/\\"\";\" > /etc/apt/apt.conf.d/000apt-cacher-ng-proxy"
236  fi
237
238  Dockerfile=$(cat << EOF
239  FROM ${DOCKER_BASE}${distro}:${img_tag}
240
241  ${PROXY}
242
243  ENV DEBIAN_FRONTEND noninteractive
244
245  RUN apt-get update && apt-get install -yy \
246      build-essential \
247      chrpath \
248      debianutils \
249      diffstat \
250      gawk \
251      git \
252      libdata-dumper-simple-perl \
253      libsdl1.2-dev \
254      libthread-queue-any-perl \
255      locales \
256      python \
257      python3 \
258      socat \
259      subversion \
260      texinfo \
261      cpio \
262      wget \
263      iputils-ping
264
265  # Set the locale
266  RUN locale-gen en_US.UTF-8
267  ENV LANG en_US.UTF-8
268  ENV LANGUAGE en_US:en
269  ENV LC_ALL en_US.UTF-8
270
271  RUN grep -q ${GROUPS} /etc/group || groupadd -g ${GROUPS} ${USER}
272  RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS} ${USER}
273
274  USER ${USER}
275  ENV HOME ${HOME}
276  RUN /bin/bash
277EOF
278)
279fi
280
281# Create the Docker run script
282export PROXY_HOST=${http_proxy/#http*:\/\/}
283export PROXY_HOST=${PROXY_HOST/%:[0-9]*}
284export PROXY_PORT=${http_proxy/#http*:\/\/*:}
285
286mkdir -p ${WORKSPACE}
287
288# Determine command for bitbake image build
289bitbake_image="obmc-phosphor-image"
290if [ $no_tar = "false" ]; then
291    bitbake_image="${bitbake_image} obmc-phosphor-debug-tarball"
292fi
293
294cat > "${WORKSPACE}"/build.sh << EOF_SCRIPT
295#!/bin/bash
296
297set -xeo pipefail
298
299# Go into the OpenBMC directory, the build will handle changing directories
300cd ${obmc_dir}
301
302# Set up proxies
303export ftp_proxy=${http_proxy}
304export http_proxy=${http_proxy}
305export https_proxy=${http_proxy}
306
307mkdir -p ${WORKSPACE}/bin
308
309# Configure proxies for BitBake
310if [[ -n "${http_proxy}" ]]; then
311
312  cat > ${WORKSPACE}/bin/git-proxy << \EOF_GIT
313  #!/bin/bash
314  # \$1 = hostname, \$2 = port
315  PROXY=${PROXY_HOST}
316  PROXY_PORT=${PROXY_PORT}
317  exec socat STDIO PROXY:\${PROXY}:\${1}:\${2},proxyport=\${PROXY_PORT}
318EOF_GIT
319
320  chmod a+x ${WORKSPACE}/bin/git-proxy
321  export PATH=${WORKSPACE}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH}
322  git config core.gitProxy git-proxy
323
324  mkdir -p ~/.subversion
325
326  cat > ~/.subversion/servers << EOF_SVN
327  [global]
328  http-proxy-host = ${PROXY_HOST}
329  http-proxy-port = ${PROXY_PORT}
330EOF_SVN
331fi
332
333# Source our build env
334${BITBAKE_CMD}
335
336# Export MACHINE name when given for build target
337if [[ -n "${MACHINE}" ]]; then
338  export MACHINE="${MACHINE}"
339fi
340
341# Custom BitBake config settings
342cat >> conf/local.conf << EOF_CONF
343BB_NUMBER_THREADS = "$(nproc)"
344PARALLEL_MAKE = "-j$(nproc)"
345INHERIT += "rm_work"
346BB_GENERATE_MIRROR_TARBALLS = "1"
347DL_DIR="${ssc_dir}/bitbake_downloads"
348SSTATE_DIR="${ssc_dir}/bitbake_sharedstatecache"
349USER_CLASSES += "buildstats"
350INHERIT_remove = "uninative"
351TMPDIR="${build_dir}"
352EOF_CONF
353
354# Kick off a build
355if [[ -n "${nice_priority}" ]]; then
356    nice -${nice_priority} bitbake ${BITBAKE_OPTS} ${bitbake_image}
357else
358    bitbake ${BITBAKE_OPTS} ${bitbake_image}
359fi
360
361# Copy internal build directory into xtrct_path directory
362if [[ ${xtrct_small_copy_dir} ]]; then
363  mkdir -p ${xtrct_path}/${xtrct_small_copy_dir}
364  timeout ${xtrct_copy_timeout} cp -r ${build_dir}/${xtrct_small_copy_dir}/* ${xtrct_path}/${xtrct_small_copy_dir}
365else
366  timeout ${xtrct_copy_timeout} cp -r ${build_dir}/* ${xtrct_path}
367fi
368
369if [[ 0 -ne $? ]]; then
370  echo "Received a non-zero exit code from timeout"
371  exit 1
372fi
373
374EOF_SCRIPT
375
376chmod a+x ${WORKSPACE}/build.sh
377
378# Give the Docker image a name based on the distro,tag,arch,and target
379img_name=${img_name:-openbmc/${distro}:${img_tag}-${target}-${ARCH}}
380
381# Build the Docker image
382docker build -t ${img_name} - <<< "${Dockerfile}"
383
384# If obmc_dir or ssc_dir are ${HOME} or a subdirectory they will not be mounted
385mount_obmc_dir="-v ""${obmc_dir}"":""${obmc_dir}"" "
386mount_ssc_dir="-v ""${ssc_dir}"":""${ssc_dir}"" "
387if [[ "${obmc_dir}" = "${HOME}/"* || "${obmc_dir}" = "${HOME}" ]];then
388mount_obmc_dir=""
389fi
390if [[ "${ssc_dir}" = "${HOME}/"* || "${ssc_dir}" = "${HOME}" ]];then
391mount_ssc_dir=""
392fi
393
394# Run the Docker container, execute the build.sh script
395docker run \
396--cap-add=sys_admin \
397--cap-add=sys_nice \
398--net=host \
399--rm=true \
400-e WORKSPACE=${WORKSPACE} \
401-w "${HOME}" \
402-v "${HOME}":"${HOME}" \
403${mount_obmc_dir} \
404${mount_ssc_dir} \
405--cpus="$num_cpu" \
406-t ${img_name} \
407${WORKSPACE}/build.sh
408
409# To maintain function of resources that used an older path, add a link
410ln -sf ${xtrct_path}/deploy ${WORKSPACE}/deploy
411
412# Timestamp for build
413echo "Build completed, $(date)"
414