1#!/usr/bin/env python3 2# 3# Re-generate container recipes 4# 5# This script uses the "lcitool" available from 6# 7# https://gitlab.com/libvirt/libvirt-ci 8# 9# Copyright (c) 2020 Red Hat Inc. 10# 11# This work is licensed under the terms of the GNU GPL, version 2 12# or (at your option) any later version. See the COPYING file in 13# the top-level directory. 14 15import sys 16import subprocess 17 18from pathlib import Path 19 20if len(sys.argv) != 1: 21 print("syntax: %s" % sys.argv[0], file=sys.stderr) 22 sys.exit(1) 23 24self_dir = Path(__file__).parent 25src_dir = self_dir.parent.parent 26dockerfiles_dir = Path(src_dir, "tests", "docker", "dockerfiles") 27 28lcitool_path = Path(self_dir, "libvirt-ci", "bin", "lcitool") 29 30lcitool_cmd = [lcitool_path, "--data-dir", self_dir] 31 32 33def atomic_write(filename, content): 34 tmp = filename.with_suffix(filename.suffix + ".tmp") 35 try: 36 with tmp.open("w") as fp: 37 print(content, file=fp, end="") 38 tmp.rename(filename) 39 except Exception as ex: 40 tmp.unlink() 41 raise 42 43 44def generate(filename, cmd, trailer): 45 print("Generate %s" % filename) 46 lcitool = subprocess.run(cmd, capture_output=True, encoding='utf8') 47 48 if lcitool.returncode != 0: 49 raise Exception("Failed to generate %s: %s" % (filename, lcitool.stderr)) 50 51 content = lcitool.stdout 52 if trailer is not None: 53 content += trailer 54 atomic_write(filename, content) 55 56# Optional user setting, this will always be the last thing added 57# so maximise the number of layers that are cached 58add_user_mapping = [ 59 "# As a final step configure the user (if env is defined)", 60 "ARG USER", 61 "ARG UID", 62 "RUN if [ \"${USER}\" ]; then \\", 63 " id ${USER} 2>/dev/null || useradd -u ${UID} -U ${USER}; fi\n" 64] 65 66def generate_dockerfile(host, target, project="qemu", cross=None, trailer=None): 67 filename = Path(src_dir, "tests", "docker", "dockerfiles", host + ".docker") 68 cmd = lcitool_cmd + ["dockerfile"] 69 if cross is not None: 70 cmd.extend(["--cross", cross]) 71 cmd.extend([target, project]) 72 73 if trailer is not None: 74 trailer += "\n".join(add_user_mapping) 75 else: 76 trailer = "\n".join(add_user_mapping) 77 78 generate(filename, cmd, trailer) 79 80 81def generate_cirrus(target, trailer=None): 82 filename = Path(src_dir, ".gitlab-ci.d", "cirrus", target + ".vars") 83 cmd = lcitool_cmd + ["variables", "--format", "shell", target, "qemu"] 84 generate(filename, cmd, trailer) 85 86 87def generate_pkglist(vm, target): 88 filename = Path(src_dir, "tests", "vm", "generated", vm + ".json") 89 cmd = lcitool_cmd + ["variables", "--format", "json", target, "qemu"] 90 generate(filename, cmd, None) 91 92 93def generate_yaml(os, target, arch, trailer=None): 94 filename = Path(src_dir, "scripts", "ci", "setup", os, f"{target}-{arch}.yaml") 95 cmd = lcitool_cmd + ["variables", "--format", "yaml", "-a", 96 arch, target, "qemu"] 97 generate(filename, cmd, trailer) 98 99 100# Netmap still needs to be manually built as it is yet to be packaged 101# into a distro. We also add cscope and gtags which are used in the CI 102# test 103debian12_extras = [ 104 "# netmap/cscope/global\n", 105 "RUN DEBIAN_FRONTEND=noninteractive eatmydata \\\n", 106 " apt install -y --no-install-recommends \\\n", 107 " cscope\\\n", 108 " global\\\n", 109 " linux-headers-generic\n", 110 "RUN git clone https://github.com/luigirizzo/netmap.git /usr/src/netmap\n", 111 "RUN cd /usr/src/netmap && git checkout v11.3\n", 112 "RUN cd /usr/src/netmap/LINUX && \\\n", 113 " ./configure --no-drivers --no-apps \\\n", 114 " --kernel-dir=$(ls -d /usr/src/linux-headers-*-$(dpkg --print-architecture)) \\\n", 115 " && make install\n", 116 "ENV QEMU_CONFIGURE_OPTS --enable-netmap\n" 117] 118 119# Based on the hub.docker.com/library/rust Dockerfiles 120fedora_rustup_nightly_extras = [ 121 "RUN dnf install -y wget\n", 122 "ENV RUSTUP_HOME=/usr/local/rustup CARGO_HOME=/usr/local/cargo\n", 123 "ENV RUSTC=/usr/local/rustup/toolchains/nightly-x86_64-unknown-linux-gnu/bin/rustc\n", 124 "RUN set -eux && \\\n", 125 " rustArch='x86_64-unknown-linux-gnu' && \\\n", 126 " rustupSha256='6aeece6993e902708983b209d04c0d1dbb14ebb405ddb87def578d41f920f56d' && \\\n", 127 ' url="https://static.rust-lang.org/rustup/archive/1.27.1/${rustArch}/rustup-init" && \\\n', 128 ' wget "$url" && \\\n', 129 ' echo "${rustupSha256} *rustup-init" | sha256sum -c - && \\\n', 130 " chmod +x rustup-init && \\\n", 131 " ./rustup-init -y --no-modify-path --profile default --default-toolchain nightly --default-host ${rustArch} && \\\n", 132 " chmod -R a+w $RUSTUP_HOME $CARGO_HOME && \\\n", 133 " /usr/local/cargo/bin/rustup --version && \\\n", 134 " /usr/local/cargo/bin/rustup run nightly rustc --version && \\\n", 135 ' test "$RUSTC" = "$(/usr/local/cargo/bin/rustup +nightly which rustc)"\n', 136 'ENV PATH=$CARGO_HOME/bin:$PATH\n', 137 'RUN /usr/local/cargo/bin/rustup run nightly cargo install bindgen-cli\n', 138] 139 140ubuntu2204_bindgen_extras = [ 141 "ENV CARGO_HOME=/usr/local/cargo\n", 142 'ENV PATH=$CARGO_HOME/bin:$PATH\n', 143 "RUN DEBIAN_FRONTEND=noninteractive eatmydata \\\n", 144 " apt install -y --no-install-recommends cargo\n", 145 'RUN cargo install bindgen-cli\n', 146] 147 148def cross_build(prefix, targets): 149 conf = "ENV QEMU_CONFIGURE_OPTS --cross-prefix=%s\n" % (prefix) 150 targets = "ENV DEF_TARGET_LIST %s\n" % (targets) 151 return "".join([conf, targets]) 152 153# 154# Update all the various build configurations. 155# Please keep each group sorted alphabetically for easy reading. 156# 157 158try: 159 # 160 # Standard native builds 161 # 162 generate_dockerfile("alpine", "alpine-319") 163 generate_dockerfile("centos9", "centos-stream-9") 164 generate_dockerfile("debian", "debian-12", 165 trailer="".join(debian12_extras)) 166 generate_dockerfile("fedora", "fedora-40") 167 generate_dockerfile("opensuse-leap", "opensuse-leap-15") 168 generate_dockerfile("ubuntu2204", "ubuntu-2204", 169 trailer="".join(ubuntu2204_bindgen_extras)) 170 171 # 172 # Non-fatal Rust-enabled build 173 # 174 generate_dockerfile("fedora-rust-nightly", "fedora-40", 175 trailer="".join(fedora_rustup_nightly_extras)) 176 177 # 178 # Cross compiling builds 179 # 180 generate_dockerfile("debian-amd64-cross", "debian-12", 181 cross="x86_64", 182 trailer=cross_build("x86_64-linux-gnu-", 183 "x86_64-softmmu," 184 "x86_64-linux-user," 185 "i386-softmmu,i386-linux-user")) 186 187 generate_dockerfile("debian-arm64-cross", "debian-12", 188 cross="aarch64", 189 trailer=cross_build("aarch64-linux-gnu-", 190 "aarch64-softmmu,aarch64-linux-user")) 191 192 generate_dockerfile("debian-armhf-cross", "debian-12", 193 cross="armv7l", 194 trailer=cross_build("arm-linux-gnueabihf-", 195 "arm-softmmu,arm-linux-user")) 196 197 generate_dockerfile("debian-i686-cross", "debian-12", 198 cross="i686", 199 trailer=cross_build("i686-linux-gnu-", 200 "x86_64-softmmu," 201 "x86_64-linux-user," 202 "i386-softmmu,i386-linux-user")) 203 204 generate_dockerfile("debian-mips64el-cross", "debian-12", 205 cross="mips64el", 206 trailer=cross_build("mips64el-linux-gnuabi64-", 207 "mips64el-softmmu,mips64el-linux-user")) 208 209 generate_dockerfile("debian-mipsel-cross", "debian-12", 210 cross="mipsel", 211 trailer=cross_build("mipsel-linux-gnu-", 212 "mipsel-softmmu,mipsel-linux-user")) 213 214 generate_dockerfile("debian-ppc64el-cross", "debian-12", 215 cross="ppc64le", 216 trailer=cross_build("powerpc64le-linux-gnu-", 217 "ppc64-softmmu,ppc64-linux-user")) 218 219 generate_dockerfile("debian-riscv64-cross", "debian-sid", 220 project="qemu-minimal", 221 cross="riscv64", 222 trailer=cross_build("riscv64-linux-gnu-", 223 "riscv64-softmmu,riscv64-linux-user")) 224 225 generate_dockerfile("debian-s390x-cross", "debian-12", 226 cross="s390x", 227 trailer=cross_build("s390x-linux-gnu-", 228 "s390x-softmmu,s390x-linux-user")) 229 230 generate_dockerfile("fedora-win64-cross", "fedora-40", 231 project='qemu,qemu-win-installer', 232 cross="mingw64", 233 trailer=cross_build("x86_64-w64-mingw32-", 234 "x86_64-softmmu")) 235 236 # 237 # Cirrus packages lists for GitLab 238 # 239 generate_cirrus("freebsd-14") 240 generate_cirrus("macos-14") 241 242 # 243 # VM packages lists 244 # 245 generate_pkglist("freebsd", "freebsd-14") 246 247 # 248 # Ansible package lists 249 # 250 generate_yaml("ubuntu", "ubuntu-2204", "aarch64") 251 generate_yaml("ubuntu", "ubuntu-2204", "s390x") 252 253 254 sys.exit(0) 255except Exception as ex: 256 print(str(ex), file=sys.stderr) 257 sys.exit(1) 258