1#!/bin/sh -e
2
3# Upload a created tarball to Coverity Scan, as per
4# https://scan.coverity.com/projects/qemu/builds/new
5
6# This work is licensed under the terms of the GNU GPL version 2,
7# or (at your option) any later version.
8# See the COPYING file in the top-level directory.
9#
10# Copyright (c) 2017-2020 Linaro Limited
11# Written by Peter Maydell
12
13# Note that this script will automatically download and
14# run the (closed-source) coverity build tools, so don't
15# use it if you don't trust them!
16
17# This script assumes that you're running it from a QEMU source
18# tree, and that tree is a fresh clean one, because we do an in-tree
19# build. (This is necessary so that the filenames that the Coverity
20# Scan server sees are relative paths that match up with the component
21# regular expressions it uses; an out-of-tree build won't work for this.)
22# The host machine should have as many of QEMU's dependencies
23# installed as possible, for maximum coverity coverage.
24
25# To do an upload you need to be a maintainer in the Coverity online
26# service, and you will need to know the "Coverity token", which is a
27# secret 8 digit hex string. You can find that from the web UI in the
28# project settings, if you have maintainer access there.
29
30# Command line options:
31#   --dry-run : run the tools, but don't actually do the upload
32#   --docker : create and work inside a docker container
33#   --update-tools-only : update the cached copy of the tools, but don't run them
34#   --tokenfile : file to read Coverity token from
35#   --version ver : specify version being analyzed (default: ask git)
36#   --description desc : specify description of this version (default: ask git)
37#   --srcdir : QEMU source tree to analyze (default: current working dir)
38#   --results-tarball : path to copy the results tarball to (default: don't
39#                       copy it anywhere, just upload it)
40#   --src-tarball : tarball to untar into src dir (default: none); this
41#                   is intended mainly for internal use by the Docker support
42#
43# User-specifiable environment variables:
44#  COVERITY_TOKEN -- Coverity token (default: looks at your
45#                    coverity.token config)
46#  COVERITY_EMAIL -- the email address to use for uploads (default:
47#                    looks at your git coverity.email or user.email config)
48#  COVERITY_BUILD_CMD -- make command (default: 'make -jN' where N is
49#                    number of CPUs as determined by 'nproc')
50#  COVERITY_TOOL_BASE -- set to directory to put coverity tools
51#                        (default: /tmp/coverity-tools)
52#
53# You must specify the token, either by environment variable or by
54# putting it in a file and using --tokenfile. Everything else has
55# a reasonable default if this is run from a git tree.
56
57check_upload_permissions() {
58    # Check whether we can do an upload to the server; will exit the script
59    # with status 1 if the check failed (usually a bad token);
60    # will exit the script with status 0 if the check indicated that we
61    # can't upload yet (ie we are at quota)
62    # Assumes that COVERITY_TOKEN, PROJNAME and DRYRUN have been initialized.
63
64    echo "Checking upload permissions..."
65
66    if ! up_perm="$(wget https://scan.coverity.com/api/upload_permitted --post-data "token=$COVERITY_TOKEN&project=$PROJNAME" -q -O -)"; then
67        echo "Coverity Scan API access denied: bad token?"
68        exit 1
69    fi
70
71    # Really up_perm is a JSON response with either
72    # {upload_permitted:true} or {next_upload_permitted_at:<date>}
73    # We do some hacky string parsing instead of properly parsing it.
74    case "$up_perm" in
75        *upload_permitted*true*)
76            echo "Coverity Scan: upload permitted"
77            ;;
78        *next_upload_permitted_at*)
79            if [ "$DRYRUN" = yes ]; then
80                echo "Coverity Scan: upload quota reached, continuing dry run"
81            else
82                echo "Coverity Scan: upload quota reached; stopping here"
83                # Exit success as this isn't a build error.
84                exit 0
85            fi
86            ;;
87        *)
88            echo "Coverity Scan upload check: unexpected result $up_perm"
89            exit 1
90            ;;
91    esac
92}
93
94
95update_coverity_tools () {
96    # Check for whether we need to download the Coverity tools
97    # (either because we don't have a copy, or because it's out of date)
98    # Assumes that COVERITY_TOOL_BASE, COVERITY_TOKEN and PROJNAME are set.
99
100    mkdir -p "$COVERITY_TOOL_BASE"
101    cd "$COVERITY_TOOL_BASE"
102
103    echo "Checking for new version of coverity build tools..."
104    wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_TOKEN&project=$PROJNAME&md5=1" -O coverity_tool.md5.new
105
106    if ! cmp -s coverity_tool.md5 coverity_tool.md5.new; then
107        # out of date md5 or no md5: download new build tool
108        # blow away the old build tool
109        echo "Downloading coverity build tools..."
110        rm -rf coverity_tool coverity_tool.tgz
111        wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_TOKEN&project=$PROJNAME" -O coverity_tool.tgz
112        if ! (cat coverity_tool.md5.new; echo "  coverity_tool.tgz") | md5sum -c --status; then
113            echo "Downloaded tarball didn't match md5sum!"
114            exit 1
115        fi
116        # extract the new one, keeping it corralled in a 'coverity_tool' directory
117        echo "Unpacking coverity build tools..."
118        mkdir -p coverity_tool
119        cd coverity_tool
120        tar xf ../coverity_tool.tgz
121        cd ..
122        mv coverity_tool.md5.new coverity_tool.md5
123    fi
124
125    rm -f coverity_tool.md5.new
126}
127
128
129# Check user-provided environment variables and arguments
130DRYRUN=no
131UPDATE_ONLY=no
132DOCKER=no
133
134while [ "$#" -ge 1 ]; do
135    case "$1" in
136        --dry-run)
137            shift
138            DRYRUN=yes
139            ;;
140        --update-tools-only)
141            shift
142            UPDATE_ONLY=yes
143            ;;
144        --version)
145            shift
146            if [ $# -eq 0 ]; then
147                echo "--version needs an argument"
148                exit 1
149            fi
150            VERSION="$1"
151            shift
152            ;;
153        --description)
154            shift
155            if [ $# -eq 0 ]; then
156                echo "--description needs an argument"
157                exit 1
158            fi
159            DESCRIPTION="$1"
160            shift
161            ;;
162        --tokenfile)
163            shift
164            if [ $# -eq 0 ]; then
165                echo "--tokenfile needs an argument"
166                exit 1
167            fi
168            COVERITY_TOKEN="$(cat "$1")"
169            shift
170            ;;
171        --srcdir)
172            shift
173            if [ $# -eq 0 ]; then
174                echo "--srcdir needs an argument"
175                exit 1
176            fi
177            SRCDIR="$1"
178            shift
179            ;;
180        --results-tarball)
181            shift
182            if [ $# -eq 0 ]; then
183                echo "--results-tarball needs an argument"
184                exit 1
185            fi
186            RESULTSTARBALL="$1"
187            shift
188            ;;
189        --src-tarball)
190            shift
191            if [ $# -eq 0 ]; then
192                echo "--src-tarball needs an argument"
193                exit 1
194            fi
195            SRCTARBALL="$1"
196            shift
197            ;;
198        --docker)
199            DOCKER=yes
200            shift
201            ;;
202        *)
203            echo "Unexpected argument '$1'"
204            exit 1
205            ;;
206    esac
207done
208
209if [ -z "$COVERITY_TOKEN" ]; then
210    COVERITY_TOKEN="$(git config coverity.token)"
211fi
212if [ -z "$COVERITY_TOKEN" ]; then
213    echo "COVERITY_TOKEN environment variable not set"
214    exit 1
215fi
216
217if [ -z "$COVERITY_BUILD_CMD" ]; then
218    NPROC=$(nproc)
219    COVERITY_BUILD_CMD="make -j$NPROC"
220    echo "COVERITY_BUILD_CMD: using default '$COVERITY_BUILD_CMD'"
221fi
222
223if [ -z "$COVERITY_TOOL_BASE" ]; then
224    echo "COVERITY_TOOL_BASE: using default /tmp/coverity-tools"
225    COVERITY_TOOL_BASE=/tmp/coverity-tools
226fi
227
228if [ -z "$SRCDIR" ]; then
229    SRCDIR="$PWD"
230fi
231
232PROJNAME=QEMU
233TARBALL=cov-int.tar.xz
234
235if [ "$UPDATE_ONLY" = yes ] && [ "$DOCKER" = yes ]; then
236    echo "Combining --docker and --update-only is not supported"
237    exit 1
238fi
239
240if [ "$UPDATE_ONLY" = yes ]; then
241    # Just do the tools update; we don't need to check whether
242    # we are in a source tree or have upload rights for this,
243    # so do it before some of the command line and source tree checks.
244    update_coverity_tools
245    exit 0
246fi
247
248if [ ! -e "$SRCDIR" ]; then
249    mkdir "$SRCDIR"
250fi
251
252cd "$SRCDIR"
253
254if [ ! -z "$SRCTARBALL" ]; then
255    echo "Untarring source tarball into $SRCDIR..."
256    tar xvf "$SRCTARBALL"
257fi
258
259echo "Checking this is a QEMU source tree..."
260if ! [ -e "$SRCDIR/VERSION" ]; then
261    echo "Not in a QEMU source tree?"
262    exit 1
263fi
264
265# Fill in defaults used by the non-update-only process
266if [ -z "$VERSION" ]; then
267    VERSION="$(git describe --always HEAD)"
268fi
269
270if [ -z "$DESCRIPTION" ]; then
271    DESCRIPTION="$(git rev-parse HEAD)"
272fi
273
274if [ -z "$COVERITY_EMAIL" ]; then
275    COVERITY_EMAIL="$(git config coverity.email)"
276fi
277if [ -z "$COVERITY_EMAIL" ]; then
278    COVERITY_EMAIL="$(git config user.email)"
279fi
280
281# Run ourselves inside docker if that's what the user wants
282if [ "$DOCKER" = yes ]; then
283    # build docker container including the coverity-scan tools
284    # Put the Coverity token into a temporary file that only
285    # we have read access to, and then pass it to docker build
286    # using --secret. This requires at least Docker 18.09.
287    # Mostly what we are trying to do here is ensure we don't leak
288    # the token into the Docker image.
289    umask 077
290    SECRETDIR=$(mktemp -d)
291    if [ -z "$SECRETDIR" ]; then
292        echo "Failed to create temporary directory"
293        exit 1
294    fi
295    trap 'rm -rf "$SECRETDIR"' INT TERM EXIT
296    echo "Created temporary directory $SECRETDIR"
297    SECRET="$SECRETDIR/token"
298    echo "$COVERITY_TOKEN" > "$SECRET"
299    echo "Building docker container..."
300    # TODO: This re-downloads the tools every time, rather than
301    # caching and reusing the image produced with the downloaded tools.
302    # Not sure why.
303    # TODO: how do you get 'docker build' to print the output of the
304    # commands it is running to its stdout? This would be useful for debug.
305    DOCKER_BUILDKIT=1 docker build -t coverity-scanner \
306                   --secret id=coverity.token,src="$SECRET" \
307                   -f scripts/coverity-scan/coverity-scan.docker \
308                   scripts/coverity-scan
309    echo "Archiving sources to be analyzed..."
310    ./scripts/archive-source.sh "$SECRETDIR/qemu-sources.tgz"
311    if [ "$DRYRUN" = yes ]; then
312        DRYRUNARG=--dry-run
313    fi
314    echo "Running scanner..."
315    # If we need to capture the output tarball, get the inner run to
316    # save it to the secrets directory so we can copy it out before the
317    # directory is cleaned up.
318    if [ ! -z "$RESULTSTARBALL" ]; then
319        RTARGS="--results-tarball /work/cov-int.tar.xz"
320    else
321        RTARGS=""
322    fi
323    # Arrange for this docker run to get access to the sources with -v.
324    # We pass through all the configuration from the outer script to the inner.
325    export COVERITY_EMAIL COVERITY_BUILD_CMD
326    docker run -it --env COVERITY_EMAIL --env COVERITY_BUILD_CMD \
327           -v "$SECRETDIR:/work" coverity-scanner \
328           ./run-coverity-scan --version "$VERSION" \
329           --description "$DESCRIPTION" $DRYRUNARG --tokenfile /work/token \
330           --srcdir /qemu --src-tarball /work/qemu-sources.tgz $RTARGS
331    if [ ! -z "$RESULTSTARBALL" ]; then
332        echo "Copying results tarball to $RESULTSTARBALL..."
333        cp "$SECRETDIR/cov-int.tar.xz" "$RESULTSTARBALL"
334    fi
335    echo "Docker work complete."
336    exit 0
337fi
338
339# Otherwise, continue with the full build and upload process.
340
341check_upload_permissions
342
343update_coverity_tools
344
345TOOLBIN="$(cd "$COVERITY_TOOL_BASE" && echo $PWD/coverity_tool/cov-analysis-*/bin)"
346
347if ! test -x "$TOOLBIN/cov-build"; then
348    echo "Couldn't find cov-build in the coverity build-tool directory??"
349    exit 1
350fi
351
352export PATH="$TOOLBIN:$PATH"
353
354cd "$SRCDIR"
355
356echo "Doing make distclean..."
357make distclean
358
359echo "Configuring..."
360# We configure with a fixed set of enables here to ensure that we don't
361# accidentally reduce the scope of the analysis by doing the build on
362# the system that's missing a dependency that we need to build part of
363# the codebase.
364./configure --disable-modules --enable-sdl --enable-gtk \
365    --enable-opengl --enable-vte --enable-gnutls \
366    --enable-nettle --enable-curses --enable-curl \
367    --audio-drv-list=oss,alsa,sdl,pa --enable-virtfs \
368    --enable-vnc --enable-vnc-sasl --enable-vnc-jpeg --enable-vnc-png \
369    --enable-xen --enable-brlapi \
370    --enable-linux-aio --enable-attr \
371    --enable-cap-ng --enable-trace-backends=log --enable-spice --enable-rbd \
372    --enable-xfsctl --enable-libusb --enable-usb-redir \
373    --enable-libiscsi --enable-libnfs --enable-seccomp \
374    --enable-tpm --enable-libssh --enable-lzo --enable-snappy --enable-bzip2 \
375    --enable-numa --enable-rdma --enable-smartcard --enable-virglrenderer \
376    --enable-mpath --enable-libxml2 --enable-glusterfs \
377    --enable-virtfs --enable-zstd
378
379echo "Making libqemustub.a..."
380make libqemustub.a
381
382echo "Running cov-build..."
383rm -rf cov-int
384mkdir cov-int
385cov-build --dir cov-int $COVERITY_BUILD_CMD
386
387echo "Creating results tarball..."
388tar cvf - cov-int | xz > "$TARBALL"
389
390if [ ! -z "$RESULTSTARBALL" ]; then
391    echo "Copying results tarball to $RESULTSTARBALL..."
392    cp "$TARBALL" "$RESULTSTARBALL"
393fi
394
395echo "Uploading results tarball..."
396
397if [ "$DRYRUN" = yes ]; then
398    echo "Dry run only, not uploading $TARBALL"
399    exit 0
400fi
401
402curl --form token="$COVERITY_TOKEN" --form email="$COVERITY_EMAIL" \
403     --form file=@"$TARBALL" --form version="$VERSION" \
404     --form description="$DESCRIPTION" \
405     https://scan.coverity.com/builds?project="$PROJNAME"
406
407echo "Done."
408