1#!/bin/bash 2############################################################################### 3# 4# This build script is for running the OpenBMC builds as Docker containers. 5# 6############################################################################### 7# 8# Script Variables: 9# build_scripts_dir The path of the openbmc-build-scripts directory. 10# Default: The directory containing this script 11# http_proxy The HTTP address of the proxy server to connect to. 12# Default: "", proxy is not setup if this is not set 13# WORKSPACE Path of the workspace directory where some intermediate 14# files and the images will be saved to. 15# Default: "~/{RandomNumber}" 16# num_cpu Number of cpu's to give bitbake, default is total amount 17# in system 18# UBUNTU_MIRROR [optional] The URL of a mirror of Ubuntu to override the 19# default ones in /etc/apt/sources.list 20# default is empty, and no mirror is used. 21# ENV_LOCAL_CONF [optional] The environment variables to inject into the 22# build, which will be written into local.conf. 23# default is empty. 24# CONTAINER_ONLY Set to "true" if you only want to build the docker 25# container. The bitbake will not occur in this case. 26# DOCKER_REG: <optional, the URL of a docker registry to utilize 27# instead of our default (public.ecr.aws/ubuntu) 28# (ex. docker.io or public.ecr.aws/docker/library) 29# EXTRA_DOCKER_RUN_ARGS: Optional, pass arguments to docker run 30# 31# Docker Image Build Variables: 32# BITBAKE_OPTS Set to "-c populate_sdk" or whatever other BitBake options 33# you'd like to pass into the build. 34# Default: "", no options set 35# build_dir Path where the actual BitBake build occurs inside the 36# container, path cannot be located on network storage. 37# Default: "$WORKSPACE/build" 38# distro The distro used as the base image for the build image: 39# fedora|ubuntu. Note that if you chose fedora, you will 40# need to also update DOCKER_REG to a supported fedora reg. 41# Default: "ubuntu" 42# img_name The name given to the target build's docker image. 43# Default: "openbmc/${distro}:${imgtag}-${target}" 44# img_tag The base docker image distro tag: 45# ubuntu: latest|16.04|14.04|trusty|xenial 46# fedora: 23|24|25 47# Default: "latest" 48# target The target we aim to build. Any system supported by 49# the openbmc/openbmc `setup` script is an option. 50# repotest is a target to specifically run the CI checks 51# Default: "qemuarm" 52# no_tar Set to true if you do not want the debug tar built 53# Default: "false" 54# nice_priority Set nice priority for bitbake command. 55# Nice: 56# Run with an adjusted niceness, which affects process 57# scheduling. Nice values range from -20 (most favorable 58# to the process) to 19 (least favorable to the process). 59# Default: "", nice is not used if nice_priority is not set 60# 61# Deployment Variables: 62# obmc_dir Path of the OpenBMC repo directory used as a reference 63# for the build inside the container. 64# Default: "${WORKSPACE}/openbmc" 65# ssc_dir Path of the OpenBMC shared directory that contains the 66# downloads dir and the sstate dir. 67# Default: "${HOME}" 68# xtrct_small_copy_dir 69# Directory within build_dir that should be copied to 70# xtrct_path. The directory and all parents up to, but not 71# including, build_dir will be copied. For example, if 72# build_dir is set to "/tmp/openbmc" and this is set to 73# "build/tmp", the directory at xtrct_path will have the 74# following directory structure: 75# xtrct_path 76# | - build 77# | - tmp 78# ... 79# Can also be set to the empty string to copy the entire 80# contents of build_dir to xtrct_path. 81# Default: "deploy/images". 82# 83############################################################################### 84# Trace bash processing. Set -e so when a step fails, we fail the build 85set -xeo pipefail 86 87# Script Variables: 88build_scripts_dir=${build_scripts_dir:-"$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"} 89http_proxy=${http_proxy:-} 90WORKSPACE=${WORKSPACE:-${HOME}/${RANDOM}${RANDOM}} 91num_cpu=${num_cpu:-$(nproc)} 92UBUNTU_MIRROR=${UBUNTU_MIRROR:-""} 93ENV_LOCAL_CONF=${ENV_LOCAL_CONF:-""} 94container_only=${CONTAINER_ONLY:-false} 95docker_reg=${DOCKER_REG:-"public.ecr.aws/ubuntu"} 96 97# Docker Image Build Variables: 98build_dir=${build_dir:-${WORKSPACE}/build} 99distro=${distro:-ubuntu} 100img_tag=${img_tag:-latest} 101target=${target:-qemuarm} 102no_tar=${no_tar:-false} 103nice_priority=${nice_priority:-} 104 105# Deployment variables 106obmc_dir=${obmc_dir:-${WORKSPACE}/openbmc} 107ssc_dir=${ssc_dir:-${HOME}} 108xtrct_small_copy_dir=${xtrct_small_copy_dir:-deploy/images} 109xtrct_path="${obmc_dir}/build/tmp" 110xtrct_copy_timeout="300" 111 112bitbake_target="obmc-phosphor-image" 113PROXY="" 114 115MIRROR="" 116if [[ -n "${UBUNTU_MIRROR}" ]]; then 117 MIRROR="RUN echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME) main restricted universe multiverse\" > /etc/apt/sources.list && \ 118 echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-updates main restricted universe multiverse\" >> /etc/apt/sources.list && \ 119 echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-security main restricted universe multiverse\" >> /etc/apt/sources.list && \ 120 echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-proposed main restricted universe multiverse\" >> /etc/apt/sources.list && \ 121 echo \"deb ${UBUNTU_MIRROR} \$(. /etc/os-release && echo \$VERSION_CODENAME)-backports main restricted universe multiverse\" >> /etc/apt/sources.list" 122fi 123 124# Timestamp for job 125echo "Build started, $(date)" 126 127# If the obmc_dir directory doesn't exist clone it in 128if [ ! -d "${obmc_dir}" ] && [ "${container_only}" = false ]; then 129 echo "Clone in openbmc master to ${obmc_dir}" 130 git clone https://github.com/openbmc/openbmc "${obmc_dir}" 131fi 132 133if [[ "$target" = repotest ]]; then 134 DOCKER_IMAGE_NAME=$(./scripts/build-unit-test-docker) 135 docker run --cap-add=sys_admin --rm=true \ 136 --network host \ 137 --privileged=true \ 138 -u "$USER" \ 139 -w "${obmc_dir}" -v "${obmc_dir}:${obmc_dir}" \ 140 -t "${DOCKER_IMAGE_NAME}" \ 141 "${obmc_dir}"/meta-phosphor/scripts/run-repotest 142 exit 143fi 144 145# Make and chown the xtrct_path directory to avoid permission errors 146if [ ! -d "${xtrct_path}" ]; then 147 mkdir -p "${xtrct_path}" 148fi 149chown "${UID}:${GROUPS[0]}" "${xtrct_path}" 150 151# Perform overrides for specific machines as required. 152DISTRO=${DISTRO:-} 153 154# Set build target and BitBake command 155MACHINE="${target}" 156BITBAKE_CMD="source ./setup ${MACHINE} ${build_dir}" 157 158# Configure Docker build 159if [[ "${distro}" == fedora ]];then 160 161 if [[ -n "${http_proxy}" ]]; then 162 PROXY="RUN echo \"proxy=${http_proxy}\" >> /etc/dnf/dnf.conf" 163 fi 164 165 Dockerfile=$(cat << EOF 166 FROM ${docker_reg}/${distro}:${img_tag} 167 168 ${PROXY} 169 170 RUN dnf --refresh install -y \ 171 bzip2 \ 172 chrpath \ 173 cpio \ 174 diffstat \ 175 file \ 176 findutils \ 177 gcc \ 178 gcc-c++ \ 179 git \ 180 lz4 \ 181 make \ 182 patch \ 183 perl-bignum \ 184 perl-Data-Dumper \ 185 perl-Thread-Queue \ 186 python3-devel \ 187 SDL-devel \ 188 socat \ 189 subversion \ 190 tar \ 191 texinfo \ 192 wget \ 193 which \ 194 file \ 195 hostname \ 196 rpcgen \ 197 glibc-langpack-en \ 198 glibc-locale-source \ 199 zstd 200 201 # Set the locale 202 ENV LANG=en_US.utf8 203 RUN localedef -f UTF-8 -i en_US en_US.UTF-8 204 205 RUN grep -q ${GROUPS[0]} /etc/group || groupadd -g ${GROUPS[0]} ${USER} 206 RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS[0]} ${USER} 207 208 USER ${USER} 209 ENV HOME ${HOME} 210EOF 211 ) 212 213elif [[ "${distro}" == ubuntu ]]; then 214 215 if [[ -n "${http_proxy}" ]]; then 216 PROXY="RUN echo \"Acquire::http::Proxy \\"\"${http_proxy}/\\"\";\" > /etc/apt/apt.conf.d/000apt-cacher-ng-proxy" 217 fi 218 219 Dockerfile=$(cat << EOF 220 FROM ${docker_reg}/${distro}:${img_tag} 221 222 ${PROXY} 223 ${MIRROR} 224 225 ENV DEBIAN_FRONTEND noninteractive 226 227 RUN apt-get update && apt-get install -yy \ 228 build-essential \ 229 chrpath \ 230 cpio \ 231 debianutils \ 232 diffstat \ 233 file \ 234 gawk \ 235 git \ 236 iputils-ping \ 237 libdata-dumper-simple-perl \ 238 lz4 \ 239 libsdl1.2-dev \ 240 libthread-queue-any-perl \ 241 locales \ 242 python3 \ 243 socat \ 244 subversion \ 245 texinfo \ 246 vim \ 247 wget \ 248 zstd 249 250 # Set the locale 251 RUN locale-gen en_US.UTF-8 252 ENV LANG en_US.UTF-8 253 ENV LANGUAGE en_US:en 254 ENV LC_ALL en_US.UTF-8 255 256 # Latest Ubuntu added a default user (ubuntu), which takes 1000 UID. 257 # If the user calling this build script happens to also have a UID of 1000 258 # then the container no longer will work. Delete the new ubuntu user 259 # so there is no conflict 260 RUN if id ubuntu > /dev/null 2>&1; then userdel -r ubuntu > /dev/null 2>&1; fi 261 RUN grep -q ${GROUPS[0]} /etc/group || groupadd -g ${GROUPS[0]} ${USER} 262 RUN grep -q ${UID} /etc/passwd || useradd -d ${HOME} -m -u ${UID} -g ${GROUPS[0]} ${USER} 263 264 USER ${USER} 265 ENV HOME ${HOME} 266EOF 267 ) 268fi 269 270# Create the Docker run script 271export PROXY_HOST=${http_proxy/#http*:\/\/} 272export PROXY_HOST=${PROXY_HOST/%:[0-9]*} 273export PROXY_PORT=${http_proxy/#http*:\/\/*:} 274 275mkdir -p "${WORKSPACE}" 276 277# Determine command for bitbake image build 278if [ "$no_tar" = "false" ]; then 279 bitbake_target="${bitbake_target} obmc-phosphor-debug-tarball" 280fi 281 282cat > "${WORKSPACE}"/build.sh << EOF_SCRIPT 283#!/bin/bash 284 285set -xeo pipefail 286 287# Go into the OpenBMC directory, the build will handle changing directories 288cd ${obmc_dir} 289 290# Set up proxies 291export ftp_proxy=${http_proxy} 292export http_proxy=${http_proxy} 293export https_proxy=${http_proxy} 294 295mkdir -p ${WORKSPACE}/bin 296 297# Configure proxies for BitBake 298if [[ -n "${http_proxy}" ]]; then 299 300 cat > ${WORKSPACE}/bin/git-proxy << \EOF_GIT 301 #!/bin/bash 302 # \$1 = hostname, \$2 = port 303 PROXY=${PROXY_HOST} 304 PROXY_PORT=${PROXY_PORT} 305 exec socat STDIO PROXY:\${PROXY}:\${1}:\${2},proxyport=\${PROXY_PORT} 306EOF_GIT 307 308 chmod a+x ${WORKSPACE}/bin/git-proxy 309 export PATH=${WORKSPACE}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH} 310 311 lock=${HOME}/build-setup.lock 312 flock \${lock} git config --global core.gitProxy ${WORKSPACE}/bin/git-proxy 313 flock \${lock} git config --global http.proxy ${http_proxy} 314 315 flock \${lock} mkdir -p ~/.subversion 316 flock \${lock} cat > ~/.subversion/servers << EOF_SVN 317 [global] 318 http-proxy-host = ${PROXY_HOST} 319 http-proxy-port = ${PROXY_PORT} 320EOF_SVN 321 322 flock \${lock} cat > ~/.wgetrc << EOF_WGETRC 323 https_proxy = ${http_proxy} 324 http_proxy = ${http_proxy} 325 use_proxy = on 326EOF_WGETRC 327 328 flock \${lock} cat > ~/.curlrc << EOF_CURLRC 329 proxy = ${PROXY_HOST}:${PROXY_PORT} 330EOF_CURLRC 331fi 332 333# Source our build env 334${BITBAKE_CMD} 335 336if [[ -z "${MACHINE}" ]]; then 337 echo "MACHINE is not configured for ${target}" 338 exit 1 339fi 340 341export MACHINE="${MACHINE}" 342if [[ -z "${DISTRO}" ]]; then 343 echo "DISTRO is not configured for ${target} so will use default" 344 unset DISTRO 345else 346 export DISTRO="${DISTRO}" 347fi 348 349# bitbake requires SDKMACHINE be x86 350export SDKMACHINE=x86_64 351 352# Custom BitBake config settings 353cat >> conf/local.conf << EOF_CONF 354BB_NUMBER_THREADS = "$num_cpu" 355PARALLEL_MAKE = "-j$num_cpu" 356INHERIT += "rm_work" 357BB_GENERATE_MIRROR_TARBALLS = "1" 358DL_DIR="${ssc_dir}/bitbake_downloads" 359SSTATE_DIR="${ssc_dir}/bitbake_sharedstatecache" 360USER_CLASSES += "buildstats" 361INHERIT:remove = "uninative" 362TMPDIR="${build_dir}" 363${ENV_LOCAL_CONF} 364EOF_CONF 365 366# Kick off a build 367if [[ -n "${nice_priority}" ]]; then 368 nice -${nice_priority} bitbake -k ${BITBAKE_OPTS} ${bitbake_target} 369else 370 bitbake -k ${BITBAKE_OPTS} ${bitbake_target} 371fi 372 373# Copy internal build directory into xtrct_path directory 374if [[ ${xtrct_small_copy_dir} ]]; then 375 mkdir -p ${xtrct_path}/${xtrct_small_copy_dir} 376 timeout ${xtrct_copy_timeout} cp -r ${build_dir}/${xtrct_small_copy_dir}/* ${xtrct_path}/${xtrct_small_copy_dir} 377else 378 timeout ${xtrct_copy_timeout} cp -r ${build_dir}/* ${xtrct_path} 379fi 380 381if [[ 0 -ne $? ]]; then 382 echo "Received a non-zero exit code from timeout" 383 exit 1 384fi 385 386EOF_SCRIPT 387 388chmod a+x "${WORKSPACE}/build.sh" 389 390# Give the Docker image a name based on the distro,tag,arch,and target 391img_name=${img_name:-openbmc/${distro}:${img_tag}-${target}} 392 393# Ensure appropriate docker build output to see progress and identify 394# any issues 395export BUILDKIT_PROGRESS=plain 396 397# Build the Docker image 398docker build --network=host -t "${img_name}" - <<< "${Dockerfile}" 399 400if [[ "$container_only" = "true" ]]; then 401 exit 0 402fi 403 404# If obmc_dir or ssc_dir are ${HOME} or a subdirectory they will not be mounted 405mount_obmc_dir="-v ""${obmc_dir}"":""${obmc_dir}"" " 406mount_ssc_dir="-v ""${ssc_dir}"":""${ssc_dir}"" " 407mount_workspace_dir="-v ""${WORKSPACE}"":""${WORKSPACE}"" " 408if [[ "${obmc_dir}" = "${HOME}/"* || "${obmc_dir}" = "${HOME}" ]];then 409 mount_obmc_dir="" 410fi 411if [[ "${ssc_dir}" = "${HOME}/"* || "${ssc_dir}" = "${HOME}" ]];then 412 mount_ssc_dir="" 413fi 414if [[ "${WORKSPACE}" = "${HOME}/"* || "${WORKSPACE}" = "${HOME}" ]];then 415 mount_workspace_dir="" 416fi 417 418# If we are building on a podman based machine, need to have this set in 419# the env to allow the home mount to work (no impact on non-podman systems) 420export PODMAN_USERNS="keep-id" 421 422# Run the Docker container, execute the build.sh script 423# shellcheck disable=SC2086 # mount commands word-split purposefully 424docker run \ 425 --cap-add=sys_admin \ 426 --cap-add=sys_nice \ 427 --net=host \ 428 --rm=true \ 429 -e WORKSPACE="${WORKSPACE}" \ 430 -w "${HOME}" \ 431 -v "${HOME}:${HOME}" \ 432 ${EXTRA_DOCKER_RUN_ARGS:-} \ 433 ${mount_obmc_dir} \ 434 ${mount_ssc_dir} \ 435 ${mount_workspace_dir} \ 436 "${img_name}" \ 437 "${WORKSPACE}/build.sh" 438 439# To maintain function of resources that used an older path, add a link 440ln -sf "${xtrct_path}/deploy" "${WORKSPACE}/deploy" 441 442# Timestamp for build 443echo "Build completed, $(date)" 444