1#!/bin/bash 2############################################################################### 3# 4# This build script is for running the OpenBMC builds as Docker containers. 5# 6############################################################################### 7# 8# Script Variables: 9# build_scripts_dir The path of the openbmc-build-scripts directory. 10# Default: The directory containing this script 11# http_proxy The HTTP address of the proxy server to connect to. 12# Default: "", proxy is not setup if this is not set 13# WORKSPACE Path of the workspace directory where some intermediate 14# files and the images will be saved to. 15# Default: "~/{RandomNumber}" 16# num_cpu Number of cpu's to give bitbake, default is total amount 17# in system 18# UBUNTU_MIRROR [optional] The URL of a mirror of Ubuntu to override the 19# default ones in /etc/apt/sources.list 20# default is empty, and no mirror is used. 21# ENV_LOCAL_CONF [optional] The environment variables to inject into the 22# build, which will be written into local.conf. 23# default is empty. 24# 25# Docker Image Build Variables: 26# BITBAKE_OPTS Set to "-c populate_sdk" or whatever other BitBake options 27# you'd like to pass into the build. 28# Default: "", no options set 29# build_dir Path where the actual BitBake build occurs inside the 30# container, path cannot be located on network storage. 31# Default: "$WORKSPACE/build" 32# distro The distro used as the base image for the build image: 33# fedora|ubuntu 34# Default: "ubuntu" 35# img_name The name given to the target build's docker image. 36# Default: "openbmc/${distro}:${imgtag}-${target}-${ARCH}" 37# img_tag The base docker image distro tag: 38# ubuntu: latest|16.04|14.04|trusty|xenial 39# fedora: 23|24|25 40# Default: "latest" 41# target The target we aim to build. Any system supported by 42# the openbmc/openbmc `setup` script is an option. 43# Default: "qemuarm" 44# no_tar Set to true if you do not want the debug tar built 45# Default: "false" 46# nice_priority Set nice priotity for bitbake command. 47# Nice: 48# Run with an adjusted niceness, which affects process 49# scheduling. Nice values range from -20 (most favorable 50# to the process) to 19 (least favorable to the process). 51# Default: "", nice is not used if nice_priority is not set 52# 53# Deployment Variables: 54# obmc_dir Path of the OpenBMC repo directory used as a reference 55# for the build inside the container. 56# Default: "${WORKSPACE}/openbmc" 57# ssc_dir Path of the OpenBMC shared directory that contains the 58# downloads dir and the sstate dir. 59# Default: "${HOME}" 60# xtrct_small_copy_dir 61# Directory within build_dir that should be copied to 62# xtrct_path. The directory and all parents up to, but not 63# including, build_dir will be copied. For example, if 64# build_dir is set to "/tmp/openbmc" and this is set to 65# "build/tmp", the directory at xtrct_path will have the 66# following directory structure: 67# xtrct_path 68# | - build 69# | - tmp 70# ... 71# Can also be set to the empty string to copy the entire 72# contents of build_dir to xtrct_path. 73# Default: "deploy/images". 74# 75############################################################################### 76# Trace bash processing. Set -e so when a step fails, we fail the build 77set -xeo pipefail 78 79# Script Variables: 80build_scripts_dir=${build_scripts_dir:-"$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"} 81http_proxy=${http_proxy:-} 82WORKSPACE=${WORKSPACE:-${HOME}/${RANDOM}${RANDOM}} 83num_cpu=${num_cpu:-$(nproc)} 84UBUNTU_MIRROR=${UBUNTU_MIRROR:-""} 85ENV_LOCAL_CONF=${ENV_LOCAL_CONF:-""} 86 87# Docker Image Build Variables: 88build_dir=${build_dir:-${WORKSPACE}/build} 89distro=${distro:-ubuntu} 90img_tag=${img_tag:-latest} 91target=${target:-qemuarm} 92no_tar=${no_tar:-false} 93nice_priority=${nice_priority:-} 94 95# Deployment variables 96obmc_dir=${obmc_dir:-${WORKSPACE}/openbmc} 97ssc_dir=${ssc_dir:-${HOME}} 98xtrct_small_copy_dir=${xtrct_small_copy_dir:-deploy/images} 99xtrct_path="${obmc_dir}/build/tmp" 100xtrct_copy_timeout="300" 101 102bitbake_target="obmc-phosphor-image" 103PROXY="" 104 105MIRROR="" 106if [[ -n "${UBUNTU_MIRROR}" ]]; then 107 MIRROR="RUN echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME) main restricted universe multiverse\" > /etc/apt/sources.list && \ 108 echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-updates main restricted universe multiverse\" >> /etc/apt/sources.list && \ 109 echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-security main restricted universe multiverse\" >> /etc/apt/sources.list && \ 110 echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-proposed main restricted universe multiverse\" >> /etc/apt/sources.list && \ 111 echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-backports main restricted universe multiverse\" >> /etc/apt/sources.list" 112fi 113 114# Determine the architecture 115ARCH=$(uname -m) 116 117# Determine the prefix of the Dockerfile's base image 118case ${ARCH} in 119 "ppc64le") 120 DOCKER_BASE="ppc64le/" 121 ;; 122 "x86_64") 123 DOCKER_BASE="" 124 ;; 125 *) 126 echo "Unsupported system architecture(${ARCH}) found for docker image" 127 exit 1 128esac 129 130# Timestamp for job 131echo "Build started, $(date)" 132 133# If the obmc_dir directory doesn't exist clone it in 134if [ ! -d "${obmc_dir}" ]; then 135 echo "Clone in openbmc master to ${obmc_dir}" 136 git clone https://github.com/openbmc/openbmc "${obmc_dir}" 137fi 138 139# Make and chown the xtrct_path directory to avoid permission errors 140if [ ! -d "${xtrct_path}" ]; then 141 mkdir -p "${xtrct_path}" 142fi 143chown "${UID}:${GROUPS[0]}" "${xtrct_path}" 144 145# Perform overrides for specific machines as required. 146case ${target} in 147 witherspoon-tacoma) 148 DISTRO="openbmc-openpower" 149 ;; 150 rainier) 151 DISTRO="openbmc-openpower" 152 ;; 153 *) 154 echo "Unspecified machine ${target}; default to local.sample.conf values." 155 ;; 156esac 157 158# Set build target and BitBake command 159MACHINE="${target}" 160BITBAKE_CMD="source ./setup ${MACHINE} ${build_dir}" 161 162# Configure Docker build 163if [[ "${distro}" == fedora ]];then 164 165 if [[ -n "${http_proxy}" ]]; then 166 PROXY="RUN echo \"proxy=${http_proxy}\" >> /etc/dnf/dnf.conf" 167 fi 168 169 Dockerfile=$(cat << EOF 170 FROM ${DOCKER_BASE}${distro}:${img_tag} 171 172 ${PROXY} 173 174 RUN dnf --refresh install -y \ 175 bzip2 \ 176 chrpath \ 177 cpio \ 178 diffstat \ 179 findutils \ 180 gcc \ 181 gcc-c++ \ 182 git \ 183 make \ 184 patch \ 185 perl-bignum \ 186 perl-Data-Dumper \ 187 perl-Thread-Queue \ 188 python-devel \ 189 python3-devel \ 190 SDL-devel \ 191 socat \ 192 subversion \ 193 tar \ 194 texinfo \ 195 wget \ 196 which \ 197 file \ 198 hostname \ 199 rpcgen \ 200 glibc-langpack-en \ 201 glibc-locale-source 202 203 # Set the locale 204 ENV LANG=en_US.utf8 205 RUN localedef -f UTF-8 -i en_US en_US.UTF-8 206 207 RUN grep -q ${GROUPS[0]} /etc/group || groupadd -g ${GROUPS[0]} ${USER} 208 RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS[0]} ${USER} 209 210 USER ${USER} 211 ENV HOME ${HOME} 212 RUN /bin/bash 213EOF 214) 215 216elif [[ "${distro}" == ubuntu ]]; then 217 218 if [[ -n "${http_proxy}" ]]; then 219 PROXY="RUN echo \"Acquire::http::Proxy \\"\"${http_proxy}/\\"\";\" > /etc/apt/apt.conf.d/000apt-cacher-ng-proxy" 220 fi 221 222 Dockerfile=$(cat << EOF 223 FROM ${DOCKER_BASE}${distro}:${img_tag} 224 225 ${PROXY} 226 ${MIRROR} 227 228 ENV DEBIAN_FRONTEND noninteractive 229 230 RUN apt-get update && apt-get install -yy \ 231 build-essential \ 232 chrpath \ 233 debianutils \ 234 diffstat \ 235 gawk \ 236 git \ 237 libdata-dumper-simple-perl \ 238 libsdl1.2-dev \ 239 libthread-queue-any-perl \ 240 locales \ 241 python \ 242 python3 \ 243 socat \ 244 subversion \ 245 texinfo \ 246 cpio \ 247 wget \ 248 iputils-ping 249 250 # Set the locale 251 RUN locale-gen en_US.UTF-8 252 ENV LANG en_US.UTF-8 253 ENV LANGUAGE en_US:en 254 ENV LC_ALL en_US.UTF-8 255 256 RUN grep -q ${GROUPS[0]} /etc/group || groupadd -g ${GROUPS[0]} ${USER} 257 RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS[0]} ${USER} 258 259 USER ${USER} 260 ENV HOME ${HOME} 261 RUN /bin/bash 262EOF 263) 264fi 265 266# Create the Docker run script 267export PROXY_HOST=${http_proxy/#http*:\/\/} 268export PROXY_HOST=${PROXY_HOST/%:[0-9]*} 269export PROXY_PORT=${http_proxy/#http*:\/\/*:} 270 271mkdir -p "${WORKSPACE}" 272 273# Determine command for bitbake image build 274if [ "$no_tar" = "false" ]; then 275 bitbake_target="${bitbake_target} obmc-phosphor-debug-tarball" 276fi 277 278cat > "${WORKSPACE}"/build.sh << EOF_SCRIPT 279#!/bin/bash 280 281set -xeo pipefail 282 283# Go into the OpenBMC directory, the build will handle changing directories 284cd ${obmc_dir} 285 286# Set up proxies 287export ftp_proxy=${http_proxy} 288export http_proxy=${http_proxy} 289export https_proxy=${http_proxy} 290 291mkdir -p ${WORKSPACE}/bin 292 293# Configure proxies for BitBake 294if [[ -n "${http_proxy}" ]]; then 295 296 cat > ${WORKSPACE}/bin/git-proxy << \EOF_GIT 297 #!/bin/bash 298 # \$1 = hostname, \$2 = port 299 PROXY=${PROXY_HOST} 300 PROXY_PORT=${PROXY_PORT} 301 exec socat STDIO PROXY:\${PROXY}:\${1}:\${2},proxyport=\${PROXY_PORT} 302EOF_GIT 303 304 chmod a+x ${WORKSPACE}/bin/git-proxy 305 export PATH=${WORKSPACE}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH} 306 307 lock=${HOME}/build-setup.lock 308 flock \${lock} git config --global core.gitProxy ${WORKSPACE}/bin/git-proxy 309 flock \${lock} git config --global http.proxy ${http_proxy} 310 311 flock \${lock} mkdir -p ~/.subversion 312 flock \${lock} cat > ~/.subversion/servers << EOF_SVN 313 [global] 314 http-proxy-host = ${PROXY_HOST} 315 http-proxy-port = ${PROXY_PORT} 316EOF_SVN 317 318 flock \${lock} cat > ~/.wgetrc << EOF_WGETRC 319 https_proxy = ${http_proxy} 320 http_proxy = ${http_proxy} 321 use_proxy = on 322EOF_WGETRC 323 324 flock \${lock} cat > ~/.curlrc << EOF_CURLRC 325 proxy = ${PROXY_HOST}:${PROXY_PORT} 326EOF_CURLRC 327fi 328 329# Source our build env 330${BITBAKE_CMD} 331 332if [[ -z "${MACHINE}" ]]; then 333 echo "MACHINE is not configured for ${target}" 334 exit 1 335fi 336 337export MACHINE="${MACHINE}" 338if [[ -z "${DISTRO}" ]]; then 339 echo "DISTRO is not configured for ${target} so will use default" 340 unset DISTRO 341else 342 export DISTRO="${DISTRO}" 343fi 344 345# bitbake requires SDKMACHINE be x86 346export SDKMACHINE=x86_64 347 348# Custom BitBake config settings 349cat >> conf/local.conf << EOF_CONF 350BB_NUMBER_THREADS = "$(nproc)" 351PARALLEL_MAKE = "-j$(nproc)" 352INHERIT += "rm_work" 353BB_GENERATE_MIRROR_TARBALLS = "1" 354DL_DIR="${ssc_dir}/bitbake_downloads" 355SSTATE_DIR="${ssc_dir}/bitbake_sharedstatecache" 356USER_CLASSES += "buildstats" 357INHERIT_remove = "uninative" 358TMPDIR="${build_dir}" 359${ENV_LOCAL_CONF} 360EOF_CONF 361 362# Kick off a build 363if [[ -n "${nice_priority}" ]]; then 364 nice -${nice_priority} bitbake ${BITBAKE_OPTS} ${bitbake_target} 365else 366 bitbake ${BITBAKE_OPTS} ${bitbake_target} 367fi 368 369# Copy internal build directory into xtrct_path directory 370if [[ ${xtrct_small_copy_dir} ]]; then 371 mkdir -p ${xtrct_path}/${xtrct_small_copy_dir} 372 timeout ${xtrct_copy_timeout} cp -r ${build_dir}/${xtrct_small_copy_dir}/* ${xtrct_path}/${xtrct_small_copy_dir} 373else 374 timeout ${xtrct_copy_timeout} cp -r ${build_dir}/* ${xtrct_path} 375fi 376 377if [[ 0 -ne $? ]]; then 378 echo "Received a non-zero exit code from timeout" 379 exit 1 380fi 381 382EOF_SCRIPT 383 384chmod a+x "${WORKSPACE}/build.sh" 385 386# Give the Docker image a name based on the distro,tag,arch,and target 387img_name=${img_name:-openbmc/${distro}:${img_tag}-${target}-${ARCH}} 388 389# Build the Docker image 390docker build -t "${img_name}" - <<< "${Dockerfile}" 391 392# If obmc_dir or ssc_dir are ${HOME} or a subdirectory they will not be mounted 393mount_obmc_dir="-v ""${obmc_dir}"":""${obmc_dir}"" " 394mount_ssc_dir="-v ""${ssc_dir}"":""${ssc_dir}"" " 395mount_workspace_dir="-v ""${WORKSPACE}"":""${WORKSPACE}"" " 396if [[ "${obmc_dir}" = "${HOME}/"* || "${obmc_dir}" = "${HOME}" ]];then 397mount_obmc_dir="" 398fi 399if [[ "${ssc_dir}" = "${HOME}/"* || "${ssc_dir}" = "${HOME}" ]];then 400mount_ssc_dir="" 401fi 402if [[ "${WORKSPACE}" = "${HOME}/"* || "${WORKSPACE}" = "${HOME}" ]];then 403mount_workspace_dir="" 404fi 405 406# Run the Docker container, execute the build.sh script 407# shellcheck disable=SC2086 # mount commands word-split purposefully 408docker run \ 409--cap-add=sys_admin \ 410--cap-add=sys_nice \ 411--net=host \ 412--rm=true \ 413-e WORKSPACE="${WORKSPACE}" \ 414-w "${HOME}" \ 415-v "${HOME}:${HOME}" \ 416${mount_obmc_dir} \ 417${mount_ssc_dir} \ 418${mount_workspace_dir} \ 419--cpus="$num_cpu" \ 420"${img_name}" \ 421"${WORKSPACE}/build.sh" 422 423# To maintain function of resources that used an older path, add a link 424ln -sf "${xtrct_path}/deploy" "${WORKSPACE}/deploy" 425 426# Timestamp for build 427echo "Build completed, $(date)" 428