1#!/usr/bin/python3 2""" 3build helper script for edk2, see 4https://gitlab.com/kraxel/edk2-build-config 5 6""" 7import os 8import sys 9import time 10import shutil 11import argparse 12import subprocess 13import configparser 14 15rebase_prefix = "" 16version_override = None 17release_date = None 18 19# pylint: disable=unused-variable 20def check_rebase(): 21 """ detect 'git rebase -x edk2-build.py master' testbuilds """ 22 global rebase_prefix 23 global version_override 24 gitdir = '.git' 25 26 if os.path.isfile(gitdir): 27 with open(gitdir, 'r', encoding = 'utf-8') as f: 28 (unused, gitdir) = f.read().split() 29 30 if not os.path.exists(f'{gitdir}/rebase-merge/msgnum'): 31 return 32 with open(f'{gitdir}/rebase-merge/msgnum', 'r', encoding = 'utf-8') as f: 33 msgnum = int(f.read()) 34 with open(f'{gitdir}/rebase-merge/end', 'r', encoding = 'utf-8') as f: 35 end = int(f.read()) 36 with open(f'{gitdir}/rebase-merge/head-name', 'r', encoding = 'utf-8') as f: 37 head = f.read().strip().split('/') 38 39 rebase_prefix = f'[ {int(msgnum/2)} / {int(end/2)} - {head[-1]} ] ' 40 if msgnum != end and not version_override: 41 # fixed version speeds up builds 42 version_override = "test-build-patch-series" 43 44def get_coredir(cfg): 45 if cfg.has_option('global', 'core'): 46 return os.path.abspath(cfg['global']['core']) 47 return os.getcwd() 48 49def get_toolchain(cfg, build): 50 if cfg.has_option(build, 'tool'): 51 return cfg[build]['tool'] 52 if cfg.has_option('global', 'tool'): 53 return cfg['global']['tool'] 54 return 'GCC5' 55 56def get_version(cfg, silent = False): 57 coredir = get_coredir(cfg) 58 if version_override: 59 version = version_override 60 if not silent: 61 print('') 62 print(f'### version [override]: {version}') 63 return version 64 if os.environ.get('RPM_PACKAGE_NAME'): 65 version = os.environ.get('RPM_PACKAGE_NAME') 66 version += '-' + os.environ.get('RPM_PACKAGE_VERSION') 67 version += '-' + os.environ.get('RPM_PACKAGE_RELEASE') 68 if not silent: 69 print('') 70 print(f'### version [rpmbuild]: {version}') 71 return version 72 if os.path.exists(coredir + '/.git'): 73 cmdline = [ 'git', 'describe', '--tags', '--abbrev=8', 74 '--match=edk2-stable*' ] 75 result = subprocess.run(cmdline, cwd = coredir, 76 stdout = subprocess.PIPE, 77 check = True) 78 version = result.stdout.decode().strip() 79 if not silent: 80 print('') 81 print(f'### version [git]: {version}') 82 return version 83 return None 84 85def pcd_string(name, value): 86 return f'{name}=L{value}\\0' 87 88def pcd_version(cfg, silent = False): 89 version = get_version(cfg, silent) 90 if version is None: 91 return [] 92 return [ '--pcd', pcd_string('PcdFirmwareVersionString', version) ] 93 94def pcd_release_date(): 95 if release_date is None: 96 return [] 97 return [ '--pcd', pcd_string('PcdFirmwareReleaseDateString', release_date) ] 98 99def build_message(line, line2 = None, silent = False): 100 if os.environ.get('TERM') in [ 'xterm', 'xterm-256color' ]: 101 # setxterm title 102 start = '\x1b]2;' 103 end = '\x07' 104 print(f'{start}{rebase_prefix}{line}{end}', end = '') 105 106 if silent: 107 print(f'### {rebase_prefix}{line}', flush = True) 108 else: 109 print('') 110 print('###') 111 print(f'### {rebase_prefix}{line}') 112 if line2: 113 print(f'### {line2}') 114 print('###', flush = True) 115 116def build_run(cmdline, name, section, silent = False, nologs = False): 117 if silent: 118 logfile = f'{section}.log' 119 if nologs: 120 print(f'### building in silent mode [no log] ...', flush = True) 121 else: 122 print(f'### building in silent mode [{logfile}] ...', flush = True) 123 start = time.time() 124 result = subprocess.run(cmdline, check = False, 125 stdout = subprocess.PIPE, 126 stderr = subprocess.STDOUT) 127 if not nologs: 128 with open(logfile, 'wb') as f: 129 f.write(result.stdout) 130 131 if result.returncode: 132 print('### BUILD FAILURE') 133 print('### cmdline') 134 print(cmdline) 135 print('### output') 136 print(result.stdout.decode()) 137 print(f'### exit code: {result.returncode}') 138 else: 139 secs = int(time.time() - start) 140 print(f'### OK ({int(secs/60)}:{secs%60:02d})') 141 else: 142 print(cmdline, flush = True) 143 result = subprocess.run(cmdline, check = False) 144 if result.returncode: 145 print(f'ERROR: {cmdline[0]} exited with {result.returncode}' 146 f' while building {name}') 147 sys.exit(result.returncode) 148 149def build_copy(plat, tgt, toolchain, dstdir, copy): 150 srcdir = f'Build/{plat}/{tgt}_{toolchain}' 151 names = copy.split() 152 srcfile = names[0] 153 if len(names) > 1: 154 dstfile = names[1] 155 else: 156 dstfile = os.path.basename(srcfile) 157 print(f'# copy: {srcdir} / {srcfile} => {dstdir} / {dstfile}') 158 159 src = srcdir + '/' + srcfile 160 dst = dstdir + '/' + dstfile 161 os.makedirs(os.path.dirname(dst), exist_ok = True) 162 shutil.copy(src, dst) 163 164def pad_file(dstdir, pad): 165 args = pad.split() 166 if len(args) < 2: 167 raise RuntimeError(f'missing arg for pad ({args})') 168 name = args[0] 169 size = args[1] 170 cmdline = [ 171 'truncate', 172 '--size', size, 173 dstdir + '/' + name, 174 ] 175 print(f'# padding: {dstdir} / {name} => {size}') 176 subprocess.run(cmdline, check = True) 177 178# pylint: disable=too-many-branches 179def build_one(cfg, build, jobs = None, silent = False, nologs = False): 180 b = cfg[build] 181 182 cmdline = [ 'build' ] 183 cmdline += [ '-t', get_toolchain(cfg, build) ] 184 cmdline += [ '-p', b['conf'] ] 185 186 if (b['conf'].startswith('OvmfPkg/') or 187 b['conf'].startswith('ArmVirtPkg/')): 188 cmdline += pcd_version(cfg, silent) 189 cmdline += pcd_release_date() 190 191 if jobs: 192 cmdline += [ '-n', jobs ] 193 for arch in b['arch'].split(): 194 cmdline += [ '-a', arch ] 195 if 'opts' in b: 196 for name in b['opts'].split(): 197 section = 'opts.' + name 198 for opt in cfg[section]: 199 cmdline += [ '-D', opt + '=' + cfg[section][opt] ] 200 if 'pcds' in b: 201 for name in b['pcds'].split(): 202 section = 'pcds.' + name 203 for pcd in cfg[section]: 204 cmdline += [ '--pcd', pcd + '=' + cfg[section][pcd] ] 205 if 'tgts' in b: 206 tgts = b['tgts'].split() 207 else: 208 tgts = [ 'DEBUG' ] 209 for tgt in tgts: 210 desc = None 211 if 'desc' in b: 212 desc = b['desc'] 213 build_message(f'building: {b["conf"]} ({b["arch"]}, {tgt})', 214 f'description: {desc}', 215 silent = silent) 216 build_run(cmdline + [ '-b', tgt ], 217 b['conf'], 218 build + '.' + tgt, 219 silent, 220 nologs) 221 222 if 'plat' in b: 223 # copy files 224 for cpy in b: 225 if not cpy.startswith('cpy'): 226 continue 227 build_copy(b['plat'], tgt, 228 get_toolchain(cfg, build), 229 b['dest'], b[cpy]) 230 # pad builds 231 for pad in b: 232 if not pad.startswith('pad'): 233 continue 234 pad_file(b['dest'], b[pad]) 235 236def build_basetools(silent = False, nologs = False): 237 build_message('building: BaseTools', silent = silent) 238 basedir = os.environ['EDK_TOOLS_PATH'] 239 cmdline = [ 'make', '-C', basedir ] 240 build_run(cmdline, 'BaseTools', 'build.basetools', silent, nologs) 241 242def binary_exists(name): 243 for pdir in os.environ['PATH'].split(':'): 244 if os.path.exists(pdir + '/' + name): 245 return True 246 return False 247 248def prepare_env(cfg, silent = False): 249 """ mimic Conf/BuildEnv.sh """ 250 workspace = os.getcwd() 251 packages = [ workspace, ] 252 path = os.environ['PATH'].split(':') 253 dirs = [ 254 'BaseTools/Bin/Linux-x86_64', 255 'BaseTools/BinWrappers/PosixLike' 256 ] 257 258 if cfg.has_option('global', 'pkgs'): 259 for pkgdir in cfg['global']['pkgs'].split(): 260 packages.append(os.path.abspath(pkgdir)) 261 coredir = get_coredir(cfg) 262 if coredir != workspace: 263 packages.append(coredir) 264 265 # add basetools to path 266 for pdir in dirs: 267 p = coredir + '/' + pdir 268 if not os.path.exists(p): 269 continue 270 if p in path: 271 continue 272 path.insert(0, p) 273 274 # run edksetup if needed 275 toolsdef = coredir + '/Conf/tools_def.txt' 276 if not os.path.exists(toolsdef): 277 os.makedirs(os.path.dirname(toolsdef), exist_ok = True) 278 build_message('running BaseTools/BuildEnv', silent = silent) 279 cmdline = [ 'bash', 'BaseTools/BuildEnv' ] 280 subprocess.run(cmdline, cwd = coredir, check = True) 281 282 # set variables 283 os.environ['PATH'] = ':'.join(path) 284 os.environ['PACKAGES_PATH'] = ':'.join(packages) 285 os.environ['WORKSPACE'] = workspace 286 os.environ['EDK_TOOLS_PATH'] = coredir + '/BaseTools' 287 os.environ['CONF_PATH'] = coredir + '/Conf' 288 os.environ['PYTHON_COMMAND'] = '/usr/bin/python3' 289 os.environ['PYTHONHASHSEED'] = '1' 290 291 # for cross builds 292 if binary_exists('arm-linux-gnueabi-gcc'): 293 # ubuntu 294 os.environ['GCC5_ARM_PREFIX'] = 'arm-linux-gnueabi-' 295 os.environ['GCC_ARM_PREFIX'] = 'arm-linux-gnueabi-' 296 elif binary_exists('arm-linux-gnu-gcc'): 297 # fedora 298 os.environ['GCC5_ARM_PREFIX'] = 'arm-linux-gnu-' 299 os.environ['GCC_ARM_PREFIX'] = 'arm-linux-gnu-' 300 if binary_exists('loongarch64-linux-gnu-gcc'): 301 os.environ['GCC5_LOONGARCH64_PREFIX'] = 'loongarch64-linux-gnu-' 302 os.environ['GCC_LOONGARCH64_PREFIX'] = 'loongarch64-linux-gnu-' 303 304 hostarch = os.uname().machine 305 if binary_exists('aarch64-linux-gnu-gcc') and hostarch != 'aarch64': 306 os.environ['GCC5_AARCH64_PREFIX'] = 'aarch64-linux-gnu-' 307 os.environ['GCC_AARCH64_PREFIX'] = 'aarch64-linux-gnu-' 308 if binary_exists('riscv64-linux-gnu-gcc') and hostarch != 'riscv64': 309 os.environ['GCC5_RISCV64_PREFIX'] = 'riscv64-linux-gnu-' 310 os.environ['GCC_RISCV64_PREFIX'] = 'riscv64-linux-gnu-' 311 if binary_exists('x86_64-linux-gnu-gcc') and hostarch != 'x86_64': 312 os.environ['GCC5_IA32_PREFIX'] = 'x86_64-linux-gnu-' 313 os.environ['GCC5_X64_PREFIX'] = 'x86_64-linux-gnu-' 314 os.environ['GCC5_BIN'] = 'x86_64-linux-gnu-' 315 os.environ['GCC_IA32_PREFIX'] = 'x86_64-linux-gnu-' 316 os.environ['GCC_X64_PREFIX'] = 'x86_64-linux-gnu-' 317 os.environ['GCC_BIN'] = 'x86_64-linux-gnu-' 318 319def build_list(cfg): 320 for build in cfg.sections(): 321 if not build.startswith('build.'): 322 continue 323 name = build.lstrip('build.') 324 desc = 'no description' 325 if 'desc' in cfg[build]: 326 desc = cfg[build]['desc'] 327 print(f'# {name:20s} - {desc}') 328 329def main(): 330 parser = argparse.ArgumentParser(prog = 'edk2-build', 331 description = 'edk2 build helper script') 332 parser.add_argument('-c', '--config', dest = 'configfile', 333 type = str, default = '.edk2.builds', metavar = 'FILE', 334 help = 'read configuration from FILE (default: .edk2.builds)') 335 parser.add_argument('-C', '--directory', dest = 'directory', type = str, 336 help = 'change to DIR before building', metavar = 'DIR') 337 parser.add_argument('-j', '--jobs', dest = 'jobs', type = str, 338 help = 'allow up to JOBS parallel build jobs', 339 metavar = 'JOBS') 340 parser.add_argument('-m', '--match', dest = 'match', 341 type = str, action = 'append', 342 help = 'only run builds matching INCLUDE (substring)', 343 metavar = 'INCLUDE') 344 parser.add_argument('-x', '--exclude', dest = 'exclude', 345 type = str, action = 'append', 346 help = 'skip builds matching EXCLUDE (substring)', 347 metavar = 'EXCLUDE') 348 parser.add_argument('-l', '--list', dest = 'list', 349 action = 'store_true', default = False, 350 help = 'list build configs available') 351 parser.add_argument('--silent', dest = 'silent', 352 action = 'store_true', default = False, 353 help = 'write build output to logfiles, ' 354 'write to console only on errors') 355 parser.add_argument('--no-logs', dest = 'nologs', 356 action = 'store_true', default = False, 357 help = 'do not write build log files (with --silent)') 358 parser.add_argument('--core', dest = 'core', type = str, metavar = 'DIR', 359 help = 'location of the core edk2 repository ' 360 '(i.e. where BuildTools are located)') 361 parser.add_argument('--pkg', '--package', dest = 'pkgs', 362 type = str, action = 'append', metavar = 'DIR', 363 help = 'location(s) of additional packages ' 364 '(can be specified multiple times)') 365 parser.add_argument('-t', '--toolchain', dest = 'toolchain', 366 type = str, metavar = 'NAME', 367 help = 'tool chain to be used to build edk2') 368 parser.add_argument('--version-override', dest = 'version_override', 369 type = str, metavar = 'VERSION', 370 help = 'set firmware build version') 371 parser.add_argument('--release-date', dest = 'release_date', 372 type = str, metavar = 'DATE', 373 help = 'set firmware build release date (in MM/DD/YYYY format)') 374 options = parser.parse_args() 375 376 if options.directory: 377 os.chdir(options.directory) 378 379 if not os.path.exists(options.configfile): 380 print(f'config file "{options.configfile}" not found') 381 return 1 382 383 cfg = configparser.ConfigParser() 384 cfg.optionxform = str 385 cfg.read(options.configfile) 386 387 if options.list: 388 build_list(cfg) 389 return 0 390 391 if not cfg.has_section('global'): 392 cfg.add_section('global') 393 if options.core: 394 cfg.set('global', 'core', options.core) 395 if options.pkgs: 396 cfg.set('global', 'pkgs', ' '.join(options.pkgs)) 397 if options.toolchain: 398 cfg.set('global', 'tool', options.toolchain) 399 400 global version_override 401 global release_date 402 check_rebase() 403 if options.version_override: 404 version_override = options.version_override 405 if options.release_date: 406 release_date = options.release_date 407 408 prepare_env(cfg, options.silent) 409 build_basetools(options.silent, options.nologs) 410 for build in cfg.sections(): 411 if not build.startswith('build.'): 412 continue 413 if options.match: 414 matching = False 415 for item in options.match: 416 if item in build: 417 matching = True 418 if not matching: 419 print(f'# skipping "{build}" (not matching "{"|".join(options.match)}")') 420 continue 421 if options.exclude: 422 exclude = False 423 for item in options.exclude: 424 if item in build: 425 print(f'# skipping "{build}" (matching "{item}")') 426 exclude = True 427 if exclude: 428 continue 429 build_one(cfg, build, options.jobs, options.silent, options.nologs) 430 431 return 0 432 433if __name__ == '__main__': 434 sys.exit(main()) 435