1#!/bin/sh -e 2 3# Upload a created tarball to Coverity Scan, as per 4# https://scan.coverity.com/projects/qemu/builds/new 5 6# This work is licensed under the terms of the GNU GPL version 2, 7# or (at your option) any later version. 8# See the COPYING file in the top-level directory. 9# 10# Copyright (c) 2017-2020 Linaro Limited 11# Written by Peter Maydell 12 13# Note that this script will automatically download and 14# run the (closed-source) coverity build tools, so don't 15# use it if you don't trust them! 16 17# This script assumes that you're running it from a QEMU source 18# tree, and that tree is a fresh clean one, because we do an in-tree 19# build. (This is necessary so that the filenames that the Coverity 20# Scan server sees are relative paths that match up with the component 21# regular expressions it uses; an out-of-tree build won't work for this.) 22# The host machine should have as many of QEMU's dependencies 23# installed as possible, for maximum coverity coverage. 24 25# To do an upload you need to be a maintainer in the Coverity online 26# service, and you will need to know the "Coverity token", which is a 27# secret 8 digit hex string. You can find that from the web UI in the 28# project settings, if you have maintainer access there. 29 30# Command line options: 31# --dry-run : run the tools, but don't actually do the upload 32# --docker : create and work inside a container 33# --docker-engine : specify the container engine to use (docker/podman/auto); 34# implies --docker 35# --update-tools-only : update the cached copy of the tools, but don't run them 36# --tokenfile : file to read Coverity token from 37# --version ver : specify version being analyzed (default: ask git) 38# --description desc : specify description of this version (default: ask git) 39# --srcdir : QEMU source tree to analyze (default: current working dir) 40# --results-tarball : path to copy the results tarball to (default: don't 41# copy it anywhere, just upload it) 42# --src-tarball : tarball to untar into src dir (default: none); this 43# is intended mainly for internal use by the Docker support 44# 45# User-specifiable environment variables: 46# COVERITY_TOKEN -- Coverity token (default: looks at your 47# coverity.token config) 48# COVERITY_EMAIL -- the email address to use for uploads (default: 49# looks at your git coverity.email or user.email config) 50# COVERITY_BUILD_CMD -- make command (default: 'make -jN' where N is 51# number of CPUs as determined by 'nproc') 52# COVERITY_TOOL_BASE -- set to directory to put coverity tools 53# (default: /tmp/coverity-tools) 54# 55# You must specify the token, either by environment variable or by 56# putting it in a file and using --tokenfile. Everything else has 57# a reasonable default if this is run from a git tree. 58 59check_upload_permissions() { 60 # Check whether we can do an upload to the server; will exit the script 61 # with status 1 if the check failed (usually a bad token); 62 # will exit the script with status 0 if the check indicated that we 63 # can't upload yet (ie we are at quota) 64 # Assumes that COVERITY_TOKEN, PROJNAME and DRYRUN have been initialized. 65 66 echo "Checking upload permissions..." 67 68 if ! up_perm="$(wget https://scan.coverity.com/api/upload_permitted --post-data "token=$COVERITY_TOKEN&project=$PROJNAME" -q -O -)"; then 69 echo "Coverity Scan API access denied: bad token?" 70 exit 1 71 fi 72 73 # Really up_perm is a JSON response with either 74 # {upload_permitted:true} or {next_upload_permitted_at:<date>} 75 # We do some hacky string parsing instead of properly parsing it. 76 case "$up_perm" in 77 *upload_permitted*true*) 78 echo "Coverity Scan: upload permitted" 79 ;; 80 *next_upload_permitted_at*) 81 if [ "$DRYRUN" = yes ]; then 82 echo "Coverity Scan: upload quota reached, continuing dry run" 83 else 84 echo "Coverity Scan: upload quota reached; stopping here" 85 # Exit success as this isn't a build error. 86 exit 0 87 fi 88 ;; 89 *) 90 echo "Coverity Scan upload check: unexpected result $up_perm" 91 exit 1 92 ;; 93 esac 94} 95 96 97update_coverity_tools () { 98 # Check for whether we need to download the Coverity tools 99 # (either because we don't have a copy, or because it's out of date) 100 # Assumes that COVERITY_TOOL_BASE, COVERITY_TOKEN and PROJNAME are set. 101 102 mkdir -p "$COVERITY_TOOL_BASE" 103 cd "$COVERITY_TOOL_BASE" 104 105 echo "Checking for new version of coverity build tools..." 106 wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_TOKEN&project=$PROJNAME&md5=1" -O coverity_tool.md5.new 107 108 if ! cmp -s coverity_tool.md5 coverity_tool.md5.new; then 109 # out of date md5 or no md5: download new build tool 110 # blow away the old build tool 111 echo "Downloading coverity build tools..." 112 rm -rf coverity_tool coverity_tool.tgz 113 wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_TOKEN&project=$PROJNAME" -O coverity_tool.tgz 114 if ! (cat coverity_tool.md5.new; echo " coverity_tool.tgz") | md5sum -c --status; then 115 echo "Downloaded tarball didn't match md5sum!" 116 exit 1 117 fi 118 # extract the new one, keeping it corralled in a 'coverity_tool' directory 119 echo "Unpacking coverity build tools..." 120 mkdir -p coverity_tool 121 cd coverity_tool 122 tar xf ../coverity_tool.tgz 123 cd .. 124 mv coverity_tool.md5.new coverity_tool.md5 125 fi 126 127 rm -f coverity_tool.md5.new 128} 129 130 131# Check user-provided environment variables and arguments 132DRYRUN=no 133UPDATE_ONLY=no 134DOCKER=no 135 136while [ "$#" -ge 1 ]; do 137 case "$1" in 138 --dry-run) 139 shift 140 DRYRUN=yes 141 ;; 142 --update-tools-only) 143 shift 144 UPDATE_ONLY=yes 145 ;; 146 --version) 147 shift 148 if [ $# -eq 0 ]; then 149 echo "--version needs an argument" 150 exit 1 151 fi 152 VERSION="$1" 153 shift 154 ;; 155 --description) 156 shift 157 if [ $# -eq 0 ]; then 158 echo "--description needs an argument" 159 exit 1 160 fi 161 DESCRIPTION="$1" 162 shift 163 ;; 164 --tokenfile) 165 shift 166 if [ $# -eq 0 ]; then 167 echo "--tokenfile needs an argument" 168 exit 1 169 fi 170 COVERITY_TOKEN="$(cat "$1")" 171 shift 172 ;; 173 --srcdir) 174 shift 175 if [ $# -eq 0 ]; then 176 echo "--srcdir needs an argument" 177 exit 1 178 fi 179 SRCDIR="$1" 180 shift 181 ;; 182 --results-tarball) 183 shift 184 if [ $# -eq 0 ]; then 185 echo "--results-tarball needs an argument" 186 exit 1 187 fi 188 RESULTSTARBALL="$1" 189 shift 190 ;; 191 --src-tarball) 192 shift 193 if [ $# -eq 0 ]; then 194 echo "--src-tarball needs an argument" 195 exit 1 196 fi 197 SRCTARBALL="$1" 198 shift 199 ;; 200 --docker) 201 DOCKER=yes 202 DOCKER_ENGINE=auto 203 shift 204 ;; 205 --docker-engine) 206 shift 207 if [ $# -eq 0 ]; then 208 echo "--docker-engine needs an argument" 209 exit 1 210 fi 211 DOCKER=yes 212 DOCKER_ENGINE="$1" 213 shift 214 ;; 215 *) 216 echo "Unexpected argument '$1'" 217 exit 1 218 ;; 219 esac 220done 221 222if [ -z "$COVERITY_TOKEN" ]; then 223 COVERITY_TOKEN="$(git config coverity.token)" 224fi 225if [ -z "$COVERITY_TOKEN" ]; then 226 echo "COVERITY_TOKEN environment variable not set" 227 exit 1 228fi 229 230if [ -z "$COVERITY_BUILD_CMD" ]; then 231 NPROC=$(nproc) 232 COVERITY_BUILD_CMD="make -j$NPROC" 233 echo "COVERITY_BUILD_CMD: using default '$COVERITY_BUILD_CMD'" 234fi 235 236if [ -z "$COVERITY_TOOL_BASE" ]; then 237 echo "COVERITY_TOOL_BASE: using default /tmp/coverity-tools" 238 COVERITY_TOOL_BASE=/tmp/coverity-tools 239fi 240 241if [ -z "$SRCDIR" ]; then 242 SRCDIR="$PWD" 243fi 244 245PROJNAME=QEMU 246TARBALL=cov-int.tar.xz 247 248if [ "$UPDATE_ONLY" = yes ] && [ "$DOCKER" = yes ]; then 249 echo "Combining --docker and --update-only is not supported" 250 exit 1 251fi 252 253if [ "$UPDATE_ONLY" = yes ]; then 254 # Just do the tools update; we don't need to check whether 255 # we are in a source tree or have upload rights for this, 256 # so do it before some of the command line and source tree checks. 257 update_coverity_tools 258 exit 0 259fi 260 261if [ ! -e "$SRCDIR" ]; then 262 mkdir "$SRCDIR" 263fi 264 265cd "$SRCDIR" 266 267if [ ! -z "$SRCTARBALL" ]; then 268 echo "Untarring source tarball into $SRCDIR..." 269 tar xvf "$SRCTARBALL" 270fi 271 272echo "Checking this is a QEMU source tree..." 273if ! [ -e "$SRCDIR/VERSION" ]; then 274 echo "Not in a QEMU source tree?" 275 exit 1 276fi 277 278# Fill in defaults used by the non-update-only process 279if [ -z "$VERSION" ]; then 280 VERSION="$(git describe --always HEAD)" 281fi 282 283if [ -z "$DESCRIPTION" ]; then 284 DESCRIPTION="$(git rev-parse HEAD)" 285fi 286 287if [ -z "$COVERITY_EMAIL" ]; then 288 COVERITY_EMAIL="$(git config coverity.email)" 289fi 290if [ -z "$COVERITY_EMAIL" ]; then 291 COVERITY_EMAIL="$(git config user.email)" 292fi 293 294# Run ourselves inside docker if that's what the user wants 295if [ "$DOCKER" = yes ]; then 296 # build docker container including the coverity-scan tools 297 # Put the Coverity token into a temporary file that only 298 # we have read access to, and then pass it to docker build 299 # using a volume. A volume is enough for the token not to 300 # leak into the Docker image. 301 umask 077 302 SECRETDIR=$(mktemp -d) 303 if [ -z "$SECRETDIR" ]; then 304 echo "Failed to create temporary directory" 305 exit 1 306 fi 307 trap 'rm -rf "$SECRETDIR"' INT TERM EXIT 308 echo "Created temporary directory $SECRETDIR" 309 SECRET="$SECRETDIR/token" 310 echo "$COVERITY_TOKEN" > "$SECRET" 311 echo "Building docker container..." 312 # TODO: This re-downloads the tools every time, rather than 313 # caching and reusing the image produced with the downloaded tools. 314 # Not sure why. 315 tests/docker/docker.py --engine ${DOCKER_ENGINE} build \ 316 -t coverity-scanner -f scripts/coverity-scan/coverity-scan.docker \ 317 -v "$SECRETDIR:/work" \ 318 --extra-files scripts/coverity-scan/run-coverity-scan 319 echo "Archiving sources to be analyzed..." 320 ./scripts/archive-source.sh "$SECRETDIR/qemu-sources.tgz" 321 if [ "$DRYRUN" = yes ]; then 322 DRYRUNARG=--dry-run 323 fi 324 echo "Running scanner..." 325 # If we need to capture the output tarball, get the inner run to 326 # save it to the secrets directory so we can copy it out before the 327 # directory is cleaned up. 328 if [ ! -z "$RESULTSTARBALL" ]; then 329 RTARGS="--results-tarball /work/cov-int.tar.xz" 330 else 331 RTARGS="" 332 fi 333 # Arrange for this docker run to get access to the sources with -v. 334 # We pass through all the configuration from the outer script to the inner. 335 export COVERITY_EMAIL COVERITY_BUILD_CMD 336 tests/docker/docker.py run -it --env COVERITY_EMAIL --env COVERITY_BUILD_CMD \ 337 -v "$SECRETDIR:/work" coverity-scanner \ 338 ./run-coverity-scan --version "$VERSION" \ 339 --description "$DESCRIPTION" $DRYRUNARG --tokenfile /work/token \ 340 --srcdir /qemu --src-tarball /work/qemu-sources.tgz $RTARGS 341 if [ ! -z "$RESULTSTARBALL" ]; then 342 echo "Copying results tarball to $RESULTSTARBALL..." 343 cp "$SECRETDIR/cov-int.tar.xz" "$RESULTSTARBALL" 344 fi 345 echo "Docker work complete." 346 exit 0 347fi 348 349# Otherwise, continue with the full build and upload process. 350 351check_upload_permissions 352 353update_coverity_tools 354 355TOOLBIN="$(cd "$COVERITY_TOOL_BASE" && echo $PWD/coverity_tool/cov-analysis-*/bin)" 356 357if ! test -x "$TOOLBIN/cov-build"; then 358 echo "Couldn't find cov-build in the coverity build-tool directory??" 359 exit 1 360fi 361 362export PATH="$TOOLBIN:$PATH" 363 364cd "$SRCDIR" 365 366echo "Doing make distclean..." 367make distclean 368 369echo "Configuring..." 370# We configure with a fixed set of enables here to ensure that we don't 371# accidentally reduce the scope of the analysis by doing the build on 372# the system that's missing a dependency that we need to build part of 373# the codebase. 374./configure --disable-modules --enable-sdl --enable-gtk \ 375 --enable-opengl --enable-vte --enable-gnutls \ 376 --enable-nettle --enable-curses --enable-curl \ 377 --audio-drv-list=oss,alsa,sdl,pa --enable-virtfs \ 378 --enable-vnc --enable-vnc-sasl --enable-vnc-jpeg --enable-vnc-png \ 379 --enable-xen --enable-brlapi \ 380 --enable-linux-aio --enable-attr \ 381 --enable-cap-ng --enable-trace-backends=log --enable-spice --enable-rbd \ 382 --enable-xfsctl --enable-libusb --enable-usb-redir \ 383 --enable-libiscsi --enable-libnfs --enable-seccomp \ 384 --enable-tpm --enable-libssh --enable-lzo --enable-snappy --enable-bzip2 \ 385 --enable-numa --enable-rdma --enable-smartcard --enable-virglrenderer \ 386 --enable-mpath --enable-libxml2 --enable-glusterfs \ 387 --enable-virtfs --enable-zstd 388 389echo "Making libqemustub.a..." 390make libqemustub.a 391 392echo "Running cov-build..." 393rm -rf cov-int 394mkdir cov-int 395cov-build --dir cov-int $COVERITY_BUILD_CMD 396 397echo "Creating results tarball..." 398tar cvf - cov-int | xz > "$TARBALL" 399 400if [ ! -z "$RESULTSTARBALL" ]; then 401 echo "Copying results tarball to $RESULTSTARBALL..." 402 cp "$TARBALL" "$RESULTSTARBALL" 403fi 404 405echo "Uploading results tarball..." 406 407if [ "$DRYRUN" = yes ]; then 408 echo "Dry run only, not uploading $TARBALL" 409 exit 0 410fi 411 412curl --form token="$COVERITY_TOKEN" --form email="$COVERITY_EMAIL" \ 413 --form file=@"$TARBALL" --form version="$VERSION" \ 414 --form description="$DESCRIPTION" \ 415 https://scan.coverity.com/builds?project="$PROJNAME" 416 417echo "Done." 418