1#!/usr/bin/env python3 2# 3# Re-generate container recipes 4# 5# This script uses the "lcitool" available from 6# 7# https://gitlab.com/libvirt/libvirt-ci 8# 9# Copyright (c) 2020 Red Hat Inc. 10# 11# This work is licensed under the terms of the GNU GPL, version 2 12# or (at your option) any later version. See the COPYING file in 13# the top-level directory. 14 15import sys 16import subprocess 17 18from pathlib import Path 19 20if len(sys.argv) != 1: 21 print("syntax: %s" % sys.argv[0], file=sys.stderr) 22 sys.exit(1) 23 24self_dir = Path(__file__).parent 25src_dir = self_dir.parent.parent 26dockerfiles_dir = Path(src_dir, "tests", "docker", "dockerfiles") 27 28lcitool_path = Path(self_dir, "libvirt-ci", "bin", "lcitool") 29 30lcitool_cmd = [lcitool_path, "--data-dir", self_dir] 31 32 33def atomic_write(filename, content): 34 tmp = filename.with_suffix(filename.suffix + ".tmp") 35 try: 36 with tmp.open("w") as fp: 37 print(content, file=fp, end="") 38 tmp.rename(filename) 39 except Exception as ex: 40 tmp.unlink() 41 raise 42 43 44def generate(filename, cmd, trailer): 45 print("Generate %s" % filename) 46 lcitool = subprocess.run(cmd, capture_output=True, encoding='utf8') 47 48 if lcitool.returncode != 0: 49 raise Exception("Failed to generate %s: %s" % (filename, lcitool.stderr)) 50 51 content = lcitool.stdout 52 if trailer is not None: 53 content += trailer 54 atomic_write(filename, content) 55 56# Optional user setting, this will always be the last thing added 57# so maximise the number of layers that are cached 58add_user_mapping = [ 59 "# As a final step configure the user (if env is defined)", 60 "ARG USER", 61 "ARG UID", 62 "RUN if [ \"${USER}\" ]; then \\", 63 " id ${USER} 2>/dev/null || useradd -u ${UID} -U ${USER}; fi\n" 64] 65 66def generate_dockerfile(host, target, project="qemu", cross=None, trailer=None, 67 enable_rust=True): 68 filename = Path(src_dir, "tests", "docker", "dockerfiles", host + ".docker") 69 cmd = lcitool_cmd + ["dockerfile"] 70 if cross is not None: 71 cmd.extend(["--cross", cross]) 72 cmd.extend([target, project]) 73 74 if trailer is not None: 75 trailer += "\n".join(add_user_mapping) 76 else: 77 trailer = "\n".join(add_user_mapping) 78 79 if enable_rust: 80 trailer += "\nENV ENABLE_RUST 1\n" 81 generate(filename, cmd, trailer) 82 83 84def generate_cirrus(target, trailer=None): 85 filename = Path(src_dir, ".gitlab-ci.d", "cirrus", target + ".vars") 86 cmd = lcitool_cmd + ["variables", "--format", "shell", target, "qemu"] 87 generate(filename, cmd, trailer) 88 89 90def generate_pkglist(vm, target): 91 filename = Path(src_dir, "tests", "vm", "generated", vm + ".json") 92 cmd = lcitool_cmd + ["variables", "--format", "json", target, "qemu"] 93 generate(filename, cmd, None) 94 95 96def generate_yaml(os, target, arch, trailer=None): 97 filename = Path(src_dir, "scripts", "ci", "setup", os, f"{target}-{arch}.yaml") 98 cmd = lcitool_cmd + ["variables", "--format", "yaml", "-a", 99 arch, target, "qemu"] 100 generate(filename, cmd, trailer) 101 102 103alpine_extras = [ 104 "# https://gitlab.alpinelinux.org/alpine/aports/-/issues/17463\n", 105 "RUN apk add clang19-libclang\n", 106] 107 108# Netmap still needs to be manually built as it is yet to be packaged 109# into a distro. We also add cscope and gtags which are used in the CI 110# test 111debian13_extras = [ 112 "# netmap/cscope/global\n", 113 "RUN DEBIAN_FRONTEND=noninteractive eatmydata \\\n", 114 " apt install -y --no-install-recommends \\\n", 115 " cscope\\\n", 116 " global\\\n", 117 " linux-headers-generic\n", 118 "RUN git clone https://github.com/luigirizzo/netmap.git /usr/src/netmap\n", 119 "RUN cd /usr/src/netmap && git checkout v11.3\n", 120 "RUN cd /usr/src/netmap/LINUX && \\\n", 121 " ./configure --no-drivers --no-apps \\\n", 122 " --kernel-dir=$(ls -d /usr/src/linux-headers-*-$(dpkg --print-architecture)) \\\n", 123 " && make install\n", 124 "ENV QEMU_CONFIGURE_OPTS=--enable-netmap\n" 125] 126 127# Based on the hub.docker.com/library/rust Dockerfiles 128fedora_rustup_nightly_extras = [ 129 "RUN dnf install -y wget\n", 130 "ENV RUSTUP_HOME=/usr/local/rustup CARGO_HOME=/usr/local/cargo\n", 131 "ENV RUSTC=/usr/local/rustup/toolchains/nightly-x86_64-unknown-linux-gnu/bin/rustc\n", 132 "ENV RUSTDOC=/usr/local/rustup/toolchains/nightly-x86_64-unknown-linux-gnu/bin/rustdoc\n", 133 "ENV CARGO=/usr/local/rustup/toolchains/nightly-x86_64-unknown-linux-gnu/bin/cargo\n", 134 "RUN set -eux && \\\n", 135 " rustArch='x86_64-unknown-linux-gnu' && \\\n", 136 " rustupSha256='6aeece6993e902708983b209d04c0d1dbb14ebb405ddb87def578d41f920f56d' && \\\n", 137 ' url="https://static.rust-lang.org/rustup/archive/1.27.1/${rustArch}/rustup-init" && \\\n', 138 ' wget "$url" && \\\n', 139 ' echo "${rustupSha256} *rustup-init" | sha256sum -c - && \\\n', 140 " chmod +x rustup-init && \\\n", 141 " ./rustup-init -y --no-modify-path --profile default --default-toolchain nightly --default-host ${rustArch} && \\\n", 142 " chmod -R a+w $RUSTUP_HOME $CARGO_HOME && \\\n", 143 " /usr/local/cargo/bin/rustup --version && \\\n", 144 " /usr/local/cargo/bin/rustup run nightly cargo --version && \\\n", 145 " /usr/local/cargo/bin/rustup run nightly rustc --version && \\\n", 146 ' test "$CARGO" = "$(/usr/local/cargo/bin/rustup +nightly which cargo)" && \\\n', 147 ' test "$RUSTDOC" = "$(/usr/local/cargo/bin/rustup +nightly which rustdoc)" && \\\n', 148 ' test "$RUSTC" = "$(/usr/local/cargo/bin/rustup +nightly which rustc)"\n', 149 'ENV PATH=$CARGO_HOME/bin:$PATH\n', 150 'RUN /usr/local/cargo/bin/rustup run nightly cargo install bindgen-cli\n', 151 'RUN $CARGO --list\n', 152] 153 154ubuntu2204_rust_extras = [ 155 "ENV RUSTC=/usr/bin/rustc-1.83\n", 156 "ENV RUSTDOC=/usr/bin/rustdoc-1.83\n", 157 "ENV CARGO_HOME=/usr/local/cargo\n", 158 'ENV PATH=$CARGO_HOME/bin:$PATH\n', 159 "RUN DEBIAN_FRONTEND=noninteractive eatmydata \\\n", 160 " apt install -y --no-install-recommends cargo\n", 161 'RUN cargo install bindgen-cli\n', 162] 163 164debian_all_test_cross_compilers = [ 165 "# extras for cross and alternate toolchains\n", 166 "RUN DEBIAN_FRONTEND=noninteractive eatmydata \\\n", 167 " apt install -y --no-install-recommends \\\n", 168 " clang\\\n", 169 " dpkg-dev\\\n", 170 " libclang-rt-dev\n", 171 "ENV AVAILABLE_COMPILERS gcc-aarch64-linux-gnu \\\n", 172 " libc6-dev-arm64-cross \\\n", 173 " gcc-arm-linux-gnueabihf \\\n", 174 " libc6-dev-armhf-cross \\\n", 175 " gcc-mips-linux-gnu \\\n", 176 " libc6-dev-mips-cross \\\n", 177 " gcc-mips64-linux-gnuabi64 \\\n", 178 " libc6-dev-mips64-cross \\\n", 179 " gcc-mips64el-linux-gnuabi64 \\\n", 180 " libc6-dev-mips64el-cross \\\n", 181 " gcc-mipsel-linux-gnu \\\n", 182 " libc6-dev-mipsel-cross \\\n", 183 " gcc-powerpc64le-linux-gnu \\\n", 184 " libc6-dev-ppc64el-cross \\\n", 185 " gcc-riscv64-linux-gnu \\\n", 186 " libc6-dev-riscv64-cross \\\n", 187 " gcc-s390x-linux-gnu \\\n", 188 " libc6-dev-s390x-cross\n", 189 "RUN if dpkg-architecture -e amd64; then \\\n", 190 " export AVAILABLE_COMPILERS=\"${AVAILABLE_COMPILERS} gcc-hppa-linux-gnu libc6-dev-hppa-cross\"; \\\n", 191 " export AVAILABLE_COMPILERS=\"${AVAILABLE_COMPILERS} gcc-m68k-linux-gnu libc6-dev-m68k-cross\"; \\\n", 192 " export AVAILABLE_COMPILERS=\"${AVAILABLE_COMPILERS} gcc-powerpc-linux-gnu libc6-dev-powerpc-cross\"; \\\n", 193 " export AVAILABLE_COMPILERS=\"${AVAILABLE_COMPILERS} gcc-powerpc64-linux-gnu libc6-dev-ppc64-cross\"; \\\n", 194 " export AVAILABLE_COMPILERS=\"${AVAILABLE_COMPILERS} gcc-sparc64-linux-gnu libc6-dev-sparc64-cross\"; \\\n", 195 "fi && \\\n", 196 "DEBIAN_FRONTEND=noninteractive eatmydata \\\n", 197 "apt install -y --no-install-recommends \\\n", 198 "${AVAILABLE_COMPILERS} && \\\n", 199 "dpkg-query --showformat '${Package}_${Version}_${Architecture}' --show > /packages.txt\n", 200 "ENV QEMU_CONFIGURE_OPTS --disable-docs\n", 201 "ENV DEF_TARGET_LIST aarch64-linux-user,arm-linux-user,hppa-linux-user,i386-linux-user,m68k-linux-user,mips-linux-user,mips64-linux-user,mips64el-linux-user,mipsel-linux-user,ppc-linux-user,ppc64-linux-user,ppc64le-linux-user,riscv64-linux-user,s390x-linux-user,sparc64-linux-user\n", 202] 203 204def cross_build(prefix, targets): 205 conf = "ENV QEMU_CONFIGURE_OPTS=--cross-prefix=%s\n" % (prefix) 206 targets = "ENV DEF_TARGET_LIST=%s\n" % (targets) 207 return "".join([conf, targets]) 208 209# 210# Update all the various build configurations. 211# Please keep each group sorted alphabetically for easy reading. 212# 213 214try: 215 # 216 # Standard native builds 217 # 218 generate_dockerfile("alpine", "alpine-321", 219 trailer="".join(alpine_extras)) 220 generate_dockerfile("centos9", "centos-stream-9") 221 generate_dockerfile("debian", "debian-13", 222 trailer="".join(debian13_extras)) 223 generate_dockerfile("fedora", "fedora-41") 224 generate_dockerfile("opensuse-leap", "opensuse-leap-15") 225 generate_dockerfile("ubuntu2204", "ubuntu-2204", 226 trailer="".join(ubuntu2204_rust_extras), 227 # https://bugs.launchpad.net/ubuntu/+source/rustc-1.83/+bug/2120318 228 enable_rust=False) 229 230 # 231 # Non-fatal Rust-enabled build 232 # 233 generate_dockerfile("fedora-rust-nightly", "fedora-41", 234 trailer="".join(fedora_rustup_nightly_extras)) 235 236 # 237 # Cross compiling builds 238 # 239 generate_dockerfile("debian-amd64-cross", "debian-13", 240 cross="x86_64", 241 trailer=cross_build("x86_64-linux-gnu-", 242 "x86_64-softmmu," 243 "x86_64-linux-user," 244 "i386-softmmu,i386-linux-user")) 245 246 generate_dockerfile("debian-arm64-cross", "debian-13", 247 cross="aarch64", 248 trailer=cross_build("aarch64-linux-gnu-", 249 "aarch64-softmmu,aarch64-linux-user")) 250 251 generate_dockerfile("debian-armhf-cross", "debian-13", 252 cross="armv7l", 253 trailer=cross_build("arm-linux-gnueabihf-", 254 "arm-softmmu,arm-linux-user")) 255 256 generate_dockerfile("debian-i686-cross", "debian-13", 257 cross="i686", 258 trailer=cross_build("i686-linux-gnu-", 259 "i386-softmmu,i386-linux-user")) 260 261 # mips no longer supported in debian-13 262 # https://www.debian.org/releases/trixie/release-notes/issues.html#mips-architectures-removed 263 generate_dockerfile("debian-mips64el-cross", "debian-12", 264 cross="mips64el", 265 trailer=cross_build("mips64el-linux-gnuabi64-", 266 "mips64el-softmmu,mips64el-linux-user")) 267 268 generate_dockerfile("debian-mipsel-cross", "debian-12", 269 cross="mipsel", 270 trailer=cross_build("mipsel-linux-gnu-", 271 "mipsel-softmmu,mipsel-linux-user")) 272 273 generate_dockerfile("debian-ppc64el-cross", "debian-13", 274 cross="ppc64le", 275 trailer=cross_build("powerpc64le-linux-gnu-", 276 "ppc64-softmmu,ppc64-linux-user")) 277 278 # while not yet a release architecture the packages are still 279 # build while part of testing 280 generate_dockerfile("debian-riscv64-cross", "debian-13", 281 cross="riscv64", 282 trailer=cross_build("riscv64-linux-gnu-", 283 "riscv64-softmmu,riscv64-linux-user")) 284 285 generate_dockerfile("debian-s390x-cross", "debian-13", 286 cross="s390x", 287 trailer=cross_build("s390x-linux-gnu-", 288 "s390x-softmmu,s390x-linux-user")) 289 290 generate_dockerfile("fedora-win64-cross", "fedora-41", 291 project='qemu,qemu-win-installer', 292 cross="mingw64", 293 trailer=cross_build("x86_64-w64-mingw32-", 294 "x86_64-softmmu"), 295 # linking with rust is buggy: 296 # https://github.com/mesonbuild/meson/pull/14991 297 enable_rust=False) 298 299 # We also generate some docker files with minimal dependencies and 300 # as many cross-compilers as Debian will package for building TCG 301 # tests. 302 303 generate_dockerfile("debian-all-test-cross", "debian-13", 304 project="qemu-minimal", 305 enable_rust=False, 306 trailer="".join(debian_all_test_cross_compilers)) 307 308 # 309 # Cirrus packages lists for GitLab 310 # 311 generate_cirrus("freebsd-14") 312 generate_cirrus("macos-14") 313 314 # 315 # VM packages lists 316 # 317 generate_pkglist("freebsd", "freebsd-14") 318 319 # 320 # Ansible package lists 321 # 322 generate_yaml("ubuntu", "ubuntu-2404", "aarch64") 323 generate_yaml("ubuntu", "ubuntu-2404", "s390x") 324 generate_yaml("debian", "debian-13", "ppc64le") 325 326 327 sys.exit(0) 328except Exception as ex: 329 print(str(ex), file=sys.stderr) 330 sys.exit(1) 331