1#!/bin/sh -e 2 3# Upload a created tarball to Coverity Scan, as per 4# https://scan.coverity.com/projects/qemu/builds/new 5 6# This work is licensed under the terms of the GNU GPL version 2, 7# or (at your option) any later version. 8# See the COPYING file in the top-level directory. 9# 10# Copyright (c) 2017-2020 Linaro Limited 11# Written by Peter Maydell 12 13# Note that this script will automatically download and 14# run the (closed-source) coverity build tools, so don't 15# use it if you don't trust them! 16 17# This script assumes that you're running it from a QEMU source 18# tree, and that tree is a fresh clean one, because we do an in-tree 19# build. (This is necessary so that the filenames that the Coverity 20# Scan server sees are relative paths that match up with the component 21# regular expressions it uses; an out-of-tree build won't work for this.) 22# The host machine should have as many of QEMU's dependencies 23# installed as possible, for maximum coverity coverage. 24 25# To do an upload you need to be a maintainer in the Coverity online 26# service, and you will need to know the "Coverity token", which is a 27# secret 8 digit hex string. You can find that from the web UI in the 28# project settings, if you have maintainer access there. 29 30# Command line options: 31# --dry-run : run the tools, but don't actually do the upload 32# --docker : create and work inside a container 33# --docker-engine : specify the container engine to use (docker/podman/auto); 34# implies --docker 35# --update-tools-only : update the cached copy of the tools, but don't run them 36# --no-update-tools : do not update the cached copy of the tools 37# --tokenfile : file to read Coverity token from 38# --version ver : specify version being analyzed (default: ask git) 39# --description desc : specify description of this version (default: ask git) 40# --srcdir : QEMU source tree to analyze (default: current working dir) 41# --results-tarball : path to copy the results tarball to (default: don't 42# copy it anywhere, just upload it) 43# --src-tarball : tarball to untar into src dir (default: none); this 44# is intended mainly for internal use by the Docker support 45# 46# User-specifiable environment variables: 47# COVERITY_TOKEN -- Coverity token (default: looks at your 48# coverity.token config) 49# COVERITY_EMAIL -- the email address to use for uploads (default: 50# looks at your git coverity.email or user.email config) 51# COVERITY_BUILD_CMD -- make command (default: 'make -jN' where N is 52# number of CPUs as determined by 'nproc') 53# COVERITY_TOOL_BASE -- set to directory to put coverity tools 54# (default: /tmp/coverity-tools) 55# 56# You must specify the token, either by environment variable or by 57# putting it in a file and using --tokenfile. Everything else has 58# a reasonable default if this is run from a git tree. 59 60check_upload_permissions() { 61 # Check whether we can do an upload to the server; will exit the script 62 # with status 1 if the check failed (usually a bad token); 63 # will exit the script with status 0 if the check indicated that we 64 # can't upload yet (ie we are at quota) 65 # Assumes that COVERITY_TOKEN, PROJNAME and DRYRUN have been initialized. 66 67 echo "Checking upload permissions..." 68 69 if ! up_perm="$(wget https://scan.coverity.com/api/upload_permitted --post-data "token=$COVERITY_TOKEN&project=$PROJNAME" -q -O -)"; then 70 echo "Coverity Scan API access denied: bad token?" 71 exit 1 72 fi 73 74 # Really up_perm is a JSON response with either 75 # {upload_permitted:true} or {next_upload_permitted_at:<date>} 76 # We do some hacky string parsing instead of properly parsing it. 77 case "$up_perm" in 78 *upload_permitted*true*) 79 echo "Coverity Scan: upload permitted" 80 ;; 81 *next_upload_permitted_at*) 82 if [ "$DRYRUN" = yes ]; then 83 echo "Coverity Scan: upload quota reached, continuing dry run" 84 else 85 echo "Coverity Scan: upload quota reached; stopping here" 86 # Exit success as this isn't a build error. 87 exit 0 88 fi 89 ;; 90 *) 91 echo "Coverity Scan upload check: unexpected result $up_perm" 92 exit 1 93 ;; 94 esac 95} 96 97 98update_coverity_tools () { 99 # Check for whether we need to download the Coverity tools 100 # (either because we don't have a copy, or because it's out of date) 101 # Assumes that COVERITY_TOOL_BASE, COVERITY_TOKEN and PROJNAME are set. 102 103 mkdir -p "$COVERITY_TOOL_BASE" 104 cd "$COVERITY_TOOL_BASE" 105 106 echo "Checking for new version of coverity build tools..." 107 wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_TOKEN&project=$PROJNAME&md5=1" -O coverity_tool.md5.new 108 109 if ! cmp -s coverity_tool.md5 coverity_tool.md5.new; then 110 # out of date md5 or no md5: download new build tool 111 # blow away the old build tool 112 echo "Downloading coverity build tools..." 113 rm -rf coverity_tool coverity_tool.tgz 114 wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_TOKEN&project=$PROJNAME" -O coverity_tool.tgz 115 if ! (cat coverity_tool.md5.new; echo " coverity_tool.tgz") | md5sum -c --status; then 116 echo "Downloaded tarball didn't match md5sum!" 117 exit 1 118 fi 119 # extract the new one, keeping it corralled in a 'coverity_tool' directory 120 echo "Unpacking coverity build tools..." 121 mkdir -p coverity_tool 122 cd coverity_tool 123 tar xf ../coverity_tool.tgz 124 cd .. 125 mv coverity_tool.md5.new coverity_tool.md5 126 fi 127 128 rm -f coverity_tool.md5.new 129} 130 131 132# Check user-provided environment variables and arguments 133DRYRUN=no 134UPDATE=yes 135DOCKER=no 136 137while [ "$#" -ge 1 ]; do 138 case "$1" in 139 --dry-run) 140 shift 141 DRYRUN=yes 142 ;; 143 --no-update-tools) 144 shift 145 UPDATE=no 146 ;; 147 --update-tools-only) 148 shift 149 UPDATE=only 150 ;; 151 --version) 152 shift 153 if [ $# -eq 0 ]; then 154 echo "--version needs an argument" 155 exit 1 156 fi 157 VERSION="$1" 158 shift 159 ;; 160 --description) 161 shift 162 if [ $# -eq 0 ]; then 163 echo "--description needs an argument" 164 exit 1 165 fi 166 DESCRIPTION="$1" 167 shift 168 ;; 169 --tokenfile) 170 shift 171 if [ $# -eq 0 ]; then 172 echo "--tokenfile needs an argument" 173 exit 1 174 fi 175 COVERITY_TOKEN="$(cat "$1")" 176 shift 177 ;; 178 --srcdir) 179 shift 180 if [ $# -eq 0 ]; then 181 echo "--srcdir needs an argument" 182 exit 1 183 fi 184 SRCDIR="$1" 185 shift 186 ;; 187 --results-tarball) 188 shift 189 if [ $# -eq 0 ]; then 190 echo "--results-tarball needs an argument" 191 exit 1 192 fi 193 RESULTSTARBALL="$1" 194 shift 195 ;; 196 --src-tarball) 197 shift 198 if [ $# -eq 0 ]; then 199 echo "--src-tarball needs an argument" 200 exit 1 201 fi 202 SRCTARBALL="$1" 203 shift 204 ;; 205 --docker) 206 DOCKER=yes 207 DOCKER_ENGINE=auto 208 shift 209 ;; 210 --docker-engine) 211 shift 212 if [ $# -eq 0 ]; then 213 echo "--docker-engine needs an argument" 214 exit 1 215 fi 216 DOCKER=yes 217 DOCKER_ENGINE="$1" 218 shift 219 ;; 220 *) 221 echo "Unexpected argument '$1'" 222 exit 1 223 ;; 224 esac 225done 226 227if [ -z "$COVERITY_TOKEN" ]; then 228 COVERITY_TOKEN="$(git config coverity.token)" 229fi 230if [ -z "$COVERITY_TOKEN" ]; then 231 echo "COVERITY_TOKEN environment variable not set" 232 exit 1 233fi 234 235if [ -z "$COVERITY_BUILD_CMD" ]; then 236 NPROC=$(nproc) 237 COVERITY_BUILD_CMD="make -j$NPROC" 238 echo "COVERITY_BUILD_CMD: using default '$COVERITY_BUILD_CMD'" 239fi 240 241if [ -z "$COVERITY_TOOL_BASE" ]; then 242 echo "COVERITY_TOOL_BASE: using default /tmp/coverity-tools" 243 COVERITY_TOOL_BASE=/tmp/coverity-tools 244fi 245 246if [ -z "$SRCDIR" ]; then 247 SRCDIR="$PWD" 248fi 249 250PROJNAME=QEMU 251TARBALL=cov-int.tar.xz 252 253if [ "$UPDATE" = only ] && [ "$DOCKER" = yes ]; then 254 echo "Combining --docker and --update-only is not supported" 255 exit 1 256fi 257 258if [ "$UPDATE" = only ]; then 259 # Just do the tools update; we don't need to check whether 260 # we are in a source tree or have upload rights for this, 261 # so do it before some of the command line and source tree checks. 262 update_coverity_tools 263 exit 0 264fi 265 266if [ ! -e "$SRCDIR" ]; then 267 mkdir "$SRCDIR" 268fi 269 270cd "$SRCDIR" 271 272if [ ! -z "$SRCTARBALL" ]; then 273 echo "Untarring source tarball into $SRCDIR..." 274 tar xvf "$SRCTARBALL" 275fi 276 277echo "Checking this is a QEMU source tree..." 278if ! [ -e "$SRCDIR/VERSION" ]; then 279 echo "Not in a QEMU source tree?" 280 exit 1 281fi 282 283# Fill in defaults used by the non-update-only process 284if [ -z "$VERSION" ]; then 285 VERSION="$(git describe --always HEAD)" 286fi 287 288if [ -z "$DESCRIPTION" ]; then 289 DESCRIPTION="$(git rev-parse HEAD)" 290fi 291 292if [ -z "$COVERITY_EMAIL" ]; then 293 COVERITY_EMAIL="$(git config coverity.email)" 294fi 295if [ -z "$COVERITY_EMAIL" ]; then 296 COVERITY_EMAIL="$(git config user.email)" 297fi 298 299# Run ourselves inside docker if that's what the user wants 300if [ "$DOCKER" = yes ]; then 301 # Put the Coverity token into a temporary file that only 302 # we have read access to, and then pass it to docker build 303 # using a volume. A volume is enough for the token not to 304 # leak into the Docker image. 305 umask 077 306 SECRETDIR=$(mktemp -d) 307 if [ -z "$SECRETDIR" ]; then 308 echo "Failed to create temporary directory" 309 exit 1 310 fi 311 trap 'rm -rf "$SECRETDIR"' INT TERM EXIT 312 echo "Created temporary directory $SECRETDIR" 313 SECRET="$SECRETDIR/token" 314 echo "$COVERITY_TOKEN" > "$SECRET" 315 if [ "$UPDATE" != no ]; then 316 # build docker container including the coverity-scan tools 317 echo "Building docker container..." 318 # TODO: This re-downloads the tools every time, rather than 319 # caching and reusing the image produced with the downloaded tools. 320 # Not sure why. 321 tests/docker/docker.py --engine ${DOCKER_ENGINE} build \ 322 -t coverity-scanner -f scripts/coverity-scan/coverity-scan.docker \ 323 -v "$SECRETDIR:/work" \ 324 --extra-files scripts/coverity-scan/run-coverity-scan 325 fi 326 echo "Archiving sources to be analyzed..." 327 ./scripts/archive-source.sh "$SECRETDIR/qemu-sources.tgz" 328 if [ "$DRYRUN" = yes ]; then 329 DRYRUNARG=--dry-run 330 fi 331 echo "Running scanner..." 332 # If we need to capture the output tarball, get the inner run to 333 # save it to the secrets directory so we can copy it out before the 334 # directory is cleaned up. 335 if [ ! -z "$RESULTSTARBALL" ]; then 336 RTARGS="--results-tarball /work/cov-int.tar.xz" 337 else 338 RTARGS="" 339 fi 340 # Arrange for this docker run to get access to the sources with -v. 341 # We pass through all the configuration from the outer script to the inner. 342 export COVERITY_EMAIL COVERITY_BUILD_CMD 343 tests/docker/docker.py run -it --env COVERITY_EMAIL --env COVERITY_BUILD_CMD \ 344 -v "$SECRETDIR:/work" coverity-scanner \ 345 ./run-coverity-scan --version "$VERSION" \ 346 --description "$DESCRIPTION" $DRYRUNARG --tokenfile /work/token \ 347 --srcdir /qemu --src-tarball /work/qemu-sources.tgz $RTARGS 348 if [ ! -z "$RESULTSTARBALL" ]; then 349 echo "Copying results tarball to $RESULTSTARBALL..." 350 cp "$SECRETDIR/cov-int.tar.xz" "$RESULTSTARBALL" 351 fi 352 echo "Docker work complete." 353 exit 0 354fi 355 356# Otherwise, continue with the full build and upload process. 357 358check_upload_permissions 359 360if [ "$UPDATE" != no ]; then 361 update_coverity_tools 362fi 363 364TOOLBIN="$(cd "$COVERITY_TOOL_BASE" && echo $PWD/coverity_tool/cov-analysis-*/bin)" 365 366if ! test -x "$TOOLBIN/cov-build"; then 367 echo "Couldn't find cov-build in the coverity build-tool directory??" 368 exit 1 369fi 370 371export PATH="$TOOLBIN:$PATH" 372 373cd "$SRCDIR" 374 375echo "Doing make distclean..." 376make distclean 377 378echo "Configuring..." 379# We configure with a fixed set of enables here to ensure that we don't 380# accidentally reduce the scope of the analysis by doing the build on 381# the system that's missing a dependency that we need to build part of 382# the codebase. 383./configure --disable-modules --enable-sdl --enable-gtk \ 384 --enable-opengl --enable-vte --enable-gnutls \ 385 --enable-nettle --enable-curses --enable-curl \ 386 --audio-drv-list=oss,alsa,sdl,pa --enable-virtfs \ 387 --enable-vnc --enable-vnc-sasl --enable-vnc-jpeg --enable-vnc-png \ 388 --enable-xen --enable-brlapi \ 389 --enable-linux-aio --enable-attr \ 390 --enable-cap-ng --enable-trace-backends=log --enable-spice --enable-rbd \ 391 --enable-xfsctl --enable-libusb --enable-usb-redir \ 392 --enable-libiscsi --enable-libnfs --enable-seccomp \ 393 --enable-tpm --enable-libssh --enable-lzo --enable-snappy --enable-bzip2 \ 394 --enable-numa --enable-rdma --enable-smartcard --enable-virglrenderer \ 395 --enable-mpath --enable-libxml2 --enable-glusterfs \ 396 --enable-virtfs --enable-zstd 397 398echo "Making libqemustub.a..." 399make libqemustub.a 400 401echo "Running cov-build..." 402rm -rf cov-int 403mkdir cov-int 404cov-build --dir cov-int $COVERITY_BUILD_CMD 405 406echo "Creating results tarball..." 407tar cvf - cov-int | xz > "$TARBALL" 408 409if [ ! -z "$RESULTSTARBALL" ]; then 410 echo "Copying results tarball to $RESULTSTARBALL..." 411 cp "$TARBALL" "$RESULTSTARBALL" 412fi 413 414echo "Uploading results tarball..." 415 416if [ "$DRYRUN" = yes ]; then 417 echo "Dry run only, not uploading $TARBALL" 418 exit 0 419fi 420 421curl --form token="$COVERITY_TOKEN" --form email="$COVERITY_EMAIL" \ 422 --form file=@"$TARBALL" --form version="$VERSION" \ 423 --form description="$DESCRIPTION" \ 424 https://scan.coverity.com/builds?project="$PROJNAME" 425 426echo "Done." 427