1#!/bin/sh -e 2 3# Upload a created tarball to Coverity Scan, as per 4# https://scan.coverity.com/projects/qemu/builds/new 5 6# This work is licensed under the terms of the GNU GPL version 2, 7# or (at your option) any later version. 8# See the COPYING file in the top-level directory. 9# 10# Copyright (c) 2017-2020 Linaro Limited 11# Written by Peter Maydell 12 13# Note that this script will automatically download and 14# run the (closed-source) coverity build tools, so don't 15# use it if you don't trust them! 16 17# This script assumes that you're running it from a QEMU source 18# tree, and that tree is a fresh clean one, because we do an in-tree 19# build. (This is necessary so that the filenames that the Coverity 20# Scan server sees are relative paths that match up with the component 21# regular expressions it uses; an out-of-tree build won't work for this.) 22# The host machine should have as many of QEMU's dependencies 23# installed as possible, for maximum coverity coverage. 24 25# To do an upload you need to be a maintainer in the Coverity online 26# service, and you will need to know the "Coverity token", which is a 27# secret 8 digit hex string. You can find that from the web UI in the 28# project settings, if you have maintainer access there. 29 30# Command line options: 31# --dry-run : run the tools, but don't actually do the upload 32# --docker : create and work inside a container 33# --docker-engine : specify the container engine to use (docker/podman/auto); 34# implies --docker 35# --update-tools-only : update the cached copy of the tools, but don't run them 36# --no-update-tools : do not update the cached copy of the tools 37# --tokenfile : file to read Coverity token from 38# --version ver : specify version being analyzed (default: ask git) 39# --description desc : specify description of this version (default: ask git) 40# --srcdir : QEMU source tree to analyze (default: current working dir) 41# --results-tarball : path to copy the results tarball to (default: don't 42# copy it anywhere, just upload it) 43# --src-tarball : tarball to untar into src dir (default: none); this 44# is intended mainly for internal use by the Docker support 45# 46# User-specifiable environment variables: 47# COVERITY_TOKEN -- Coverity token (default: looks at your 48# coverity.token config) 49# COVERITY_EMAIL -- the email address to use for uploads (default: 50# looks at your git coverity.email or user.email config) 51# COVERITY_BUILD_CMD -- make command (default: 'make -jN' where N is 52# number of CPUs as determined by 'nproc') 53# COVERITY_TOOL_BASE -- set to directory to put coverity tools 54# (default: /tmp/coverity-tools) 55# 56# You must specify the token, either by environment variable or by 57# putting it in a file and using --tokenfile. Everything else has 58# a reasonable default if this is run from a git tree. 59 60check_upload_permissions() { 61 # Check whether we can do an upload to the server; will exit the script 62 # with status 1 if the check failed (usually a bad token); 63 # will exit the script with status 0 if the check indicated that we 64 # can't upload yet (ie we are at quota) 65 # Assumes that COVERITY_TOKEN, PROJNAME and DRYRUN have been initialized. 66 67 echo "Checking upload permissions..." 68 69 if ! up_perm="$(wget https://scan.coverity.com/api/upload_permitted --post-data "token=$COVERITY_TOKEN&project=$PROJNAME" -q -O -)"; then 70 echo "Coverity Scan API access denied: bad token?" 71 exit 1 72 fi 73 74 # Really up_perm is a JSON response with either 75 # {upload_permitted:true} or {next_upload_permitted_at:<date>} 76 # We do some hacky string parsing instead of properly parsing it. 77 case "$up_perm" in 78 *upload_permitted*true*) 79 echo "Coverity Scan: upload permitted" 80 ;; 81 *next_upload_permitted_at*) 82 if [ "$DRYRUN" = yes ]; then 83 echo "Coverity Scan: upload quota reached, continuing dry run" 84 else 85 echo "Coverity Scan: upload quota reached; stopping here" 86 # Exit success as this isn't a build error. 87 exit 0 88 fi 89 ;; 90 *) 91 echo "Coverity Scan upload check: unexpected result $up_perm" 92 exit 1 93 ;; 94 esac 95} 96 97 98update_coverity_tools () { 99 # Check for whether we need to download the Coverity tools 100 # (either because we don't have a copy, or because it's out of date) 101 # Assumes that COVERITY_TOOL_BASE, COVERITY_TOKEN and PROJNAME are set. 102 103 mkdir -p "$COVERITY_TOOL_BASE" 104 cd "$COVERITY_TOOL_BASE" 105 106 echo "Checking for new version of coverity build tools..." 107 wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_TOKEN&project=$PROJNAME&md5=1" -O coverity_tool.md5.new 108 109 if ! cmp -s coverity_tool.md5 coverity_tool.md5.new; then 110 # out of date md5 or no md5: download new build tool 111 # blow away the old build tool 112 echo "Downloading coverity build tools..." 113 rm -rf coverity_tool coverity_tool.tgz 114 wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_TOKEN&project=$PROJNAME" -O coverity_tool.tgz 115 if ! (cat coverity_tool.md5.new; echo " coverity_tool.tgz") | md5sum -c --status; then 116 echo "Downloaded tarball didn't match md5sum!" 117 exit 1 118 fi 119 120 if [ "$DOCKER" != yes ]; then 121 # extract the new one, keeping it corralled in a 'coverity_tool' directory 122 echo "Unpacking coverity build tools..." 123 mkdir -p coverity_tool 124 cd coverity_tool 125 tar xf ../coverity_tool.tgz 126 cd .. 127 mv coverity_tool.md5.new coverity_tool.md5 128 fi 129 fi 130 rm -f coverity_tool.md5.new 131} 132 133 134# Check user-provided environment variables and arguments 135DRYRUN=no 136UPDATE=yes 137DOCKER=no 138 139while [ "$#" -ge 1 ]; do 140 case "$1" in 141 --dry-run) 142 shift 143 DRYRUN=yes 144 ;; 145 --no-update-tools) 146 shift 147 UPDATE=no 148 ;; 149 --update-tools-only) 150 shift 151 UPDATE=only 152 ;; 153 --version) 154 shift 155 if [ $# -eq 0 ]; then 156 echo "--version needs an argument" 157 exit 1 158 fi 159 VERSION="$1" 160 shift 161 ;; 162 --description) 163 shift 164 if [ $# -eq 0 ]; then 165 echo "--description needs an argument" 166 exit 1 167 fi 168 DESCRIPTION="$1" 169 shift 170 ;; 171 --tokenfile) 172 shift 173 if [ $# -eq 0 ]; then 174 echo "--tokenfile needs an argument" 175 exit 1 176 fi 177 COVERITY_TOKEN="$(cat "$1")" 178 shift 179 ;; 180 --srcdir) 181 shift 182 if [ $# -eq 0 ]; then 183 echo "--srcdir needs an argument" 184 exit 1 185 fi 186 SRCDIR="$1" 187 shift 188 ;; 189 --results-tarball) 190 shift 191 if [ $# -eq 0 ]; then 192 echo "--results-tarball needs an argument" 193 exit 1 194 fi 195 RESULTSTARBALL="$1" 196 shift 197 ;; 198 --src-tarball) 199 shift 200 if [ $# -eq 0 ]; then 201 echo "--src-tarball needs an argument" 202 exit 1 203 fi 204 SRCTARBALL="$1" 205 shift 206 ;; 207 --docker) 208 DOCKER=yes 209 DOCKER_ENGINE=auto 210 shift 211 ;; 212 --docker-engine) 213 shift 214 if [ $# -eq 0 ]; then 215 echo "--docker-engine needs an argument" 216 exit 1 217 fi 218 DOCKER=yes 219 DOCKER_ENGINE="$1" 220 shift 221 ;; 222 *) 223 echo "Unexpected argument '$1'" 224 exit 1 225 ;; 226 esac 227done 228 229if [ -z "$COVERITY_TOKEN" ]; then 230 COVERITY_TOKEN="$(git config coverity.token)" 231fi 232if [ -z "$COVERITY_TOKEN" ]; then 233 echo "COVERITY_TOKEN environment variable not set" 234 exit 1 235fi 236 237if [ -z "$COVERITY_BUILD_CMD" ]; then 238 NPROC=$(nproc) 239 COVERITY_BUILD_CMD="make -j$NPROC" 240 echo "COVERITY_BUILD_CMD: using default '$COVERITY_BUILD_CMD'" 241fi 242 243if [ -z "$COVERITY_TOOL_BASE" ]; then 244 echo "COVERITY_TOOL_BASE: using default /tmp/coverity-tools" 245 COVERITY_TOOL_BASE=/tmp/coverity-tools 246fi 247 248if [ -z "$SRCDIR" ]; then 249 SRCDIR="$PWD" 250fi 251 252PROJNAME=QEMU 253TARBALL=cov-int.tar.xz 254 255if [ "$UPDATE" = only ] && [ "$DOCKER" = yes ]; then 256 echo "Combining --docker and --update-only is not supported" 257 exit 1 258fi 259 260if [ "$UPDATE" = only ]; then 261 # Just do the tools update; we don't need to check whether 262 # we are in a source tree or have upload rights for this, 263 # so do it before some of the command line and source tree checks. 264 update_coverity_tools 265 exit 0 266fi 267 268if [ ! -e "$SRCDIR" ]; then 269 mkdir "$SRCDIR" 270fi 271 272cd "$SRCDIR" 273 274if [ ! -z "$SRCTARBALL" ]; then 275 echo "Untarring source tarball into $SRCDIR..." 276 tar xvf "$SRCTARBALL" 277fi 278 279echo "Checking this is a QEMU source tree..." 280if ! [ -e "$SRCDIR/VERSION" ]; then 281 echo "Not in a QEMU source tree?" 282 exit 1 283fi 284 285# Fill in defaults used by the non-update-only process 286if [ -z "$VERSION" ]; then 287 VERSION="$(git describe --always HEAD)" 288fi 289 290if [ -z "$DESCRIPTION" ]; then 291 DESCRIPTION="$(git rev-parse HEAD)" 292fi 293 294if [ -z "$COVERITY_EMAIL" ]; then 295 COVERITY_EMAIL="$(git config coverity.email)" 296fi 297if [ -z "$COVERITY_EMAIL" ]; then 298 COVERITY_EMAIL="$(git config user.email)" 299fi 300 301# Otherwise, continue with the full build and upload process. 302 303check_upload_permissions 304 305if [ "$UPDATE" != no ]; then 306 update_coverity_tools 307fi 308 309# Run ourselves inside docker if that's what the user wants 310if [ "$DOCKER" = yes ]; then 311 # Put the Coverity token into a temporary file that only 312 # we have read access to, and then pass it to docker build 313 # using a volume. A volume is enough for the token not to 314 # leak into the Docker image. 315 umask 077 316 SECRETDIR=$(mktemp -d) 317 if [ -z "$SECRETDIR" ]; then 318 echo "Failed to create temporary directory" 319 exit 1 320 fi 321 trap 'rm -rf "$SECRETDIR"' INT TERM EXIT 322 echo "Created temporary directory $SECRETDIR" 323 SECRET="$SECRETDIR/token" 324 echo "$COVERITY_TOKEN" > "$SECRET" 325 if [ "$UPDATE" != no ]; then 326 # build docker container including the coverity-scan tools 327 echo "Building docker container..." 328 # TODO: This re-unpacks the tools every time, rather than caching 329 # and reusing the image produced by the COPY of the .tgz file. 330 # Not sure why. 331 tests/docker/docker.py --engine ${DOCKER_ENGINE} build \ 332 -t coverity-scanner -f scripts/coverity-scan/coverity-scan.docker \ 333 --extra-files scripts/coverity-scan/run-coverity-scan \ 334 "$COVERITY_TOOL_BASE"/coverity_tool.tgz 335 fi 336 echo "Archiving sources to be analyzed..." 337 ./scripts/archive-source.sh "$SECRETDIR/qemu-sources.tgz" 338 ARGS="--no-update-tools" 339 if [ "$DRYRUN" = yes ]; then 340 ARGS="$ARGS --dry-run" 341 fi 342 echo "Running scanner..." 343 # If we need to capture the output tarball, get the inner run to 344 # save it to the secrets directory so we can copy it out before the 345 # directory is cleaned up. 346 if [ ! -z "$RESULTSTARBALL" ]; then 347 ARGS="$ARGS --results-tarball /work/cov-int.tar.xz" 348 fi 349 # Arrange for this docker run to get access to the sources with -v. 350 # We pass through all the configuration from the outer script to the inner. 351 export COVERITY_EMAIL COVERITY_BUILD_CMD 352 tests/docker/docker.py run -it --env COVERITY_EMAIL --env COVERITY_BUILD_CMD \ 353 -v "$SECRETDIR:/work" coverity-scanner \ 354 ./run-coverity-scan --version "$VERSION" \ 355 --description "$DESCRIPTION" $ARGS --tokenfile /work/token \ 356 --srcdir /qemu --src-tarball /work/qemu-sources.tgz 357 if [ ! -z "$RESULTSTARBALL" ]; then 358 echo "Copying results tarball to $RESULTSTARBALL..." 359 cp "$SECRETDIR/cov-int.tar.xz" "$RESULTSTARBALL" 360 fi 361 echo "Docker work complete." 362 exit 0 363fi 364 365TOOLBIN="$(cd "$COVERITY_TOOL_BASE" && echo $PWD/coverity_tool/cov-analysis-*/bin)" 366 367if ! test -x "$TOOLBIN/cov-build"; then 368 echo "Couldn't find cov-build in the coverity build-tool directory??" 369 exit 1 370fi 371 372export PATH="$TOOLBIN:$PATH" 373 374cd "$SRCDIR" 375 376echo "Doing make distclean..." 377make distclean 378 379echo "Configuring..." 380# We configure with a fixed set of enables here to ensure that we don't 381# accidentally reduce the scope of the analysis by doing the build on 382# the system that's missing a dependency that we need to build part of 383# the codebase. 384./configure --disable-modules --enable-sdl --enable-gtk \ 385 --enable-opengl --enable-vte --enable-gnutls \ 386 --enable-nettle --enable-curses --enable-curl \ 387 --audio-drv-list=oss,alsa,sdl,pa --enable-virtfs \ 388 --enable-vnc --enable-vnc-sasl --enable-vnc-jpeg --enable-vnc-png \ 389 --enable-xen --enable-brlapi \ 390 --enable-linux-aio --enable-attr \ 391 --enable-cap-ng --enable-trace-backends=log --enable-spice --enable-rbd \ 392 --enable-xfsctl --enable-libusb --enable-usb-redir \ 393 --enable-libiscsi --enable-libnfs --enable-seccomp \ 394 --enable-tpm --enable-libssh --enable-lzo --enable-snappy --enable-bzip2 \ 395 --enable-numa --enable-rdma --enable-smartcard --enable-virglrenderer \ 396 --enable-mpath --enable-libxml2 --enable-glusterfs \ 397 --enable-virtfs --enable-zstd 398 399echo "Making libqemustub.a..." 400make libqemustub.a 401 402echo "Running cov-build..." 403rm -rf cov-int 404mkdir cov-int 405cov-build --dir cov-int $COVERITY_BUILD_CMD 406 407echo "Creating results tarball..." 408tar cvf - cov-int | xz > "$TARBALL" 409 410if [ ! -z "$RESULTSTARBALL" ]; then 411 echo "Copying results tarball to $RESULTSTARBALL..." 412 cp "$TARBALL" "$RESULTSTARBALL" 413fi 414 415echo "Uploading results tarball..." 416 417if [ "$DRYRUN" = yes ]; then 418 echo "Dry run only, not uploading $TARBALL" 419 exit 0 420fi 421 422curl --form token="$COVERITY_TOKEN" --form email="$COVERITY_EMAIL" \ 423 --form file=@"$TARBALL" --form version="$VERSION" \ 424 --form description="$DESCRIPTION" \ 425 https://scan.coverity.com/builds?project="$PROJNAME" 426 427echo "Done." 428