xref: /openbmc/qemu/roms/edk2-build.py (revision 2d708164) 
1#!/usr/bin/python3
2"""
3build helper script for edk2, see
4https://gitlab.com/kraxel/edk2-build-config
5
6"""
7import os
8import sys
9import shutil
10import argparse
11import subprocess
12import configparser
13
14rebase_prefix    = ""
15version_override = None
16release_date     = None
17
18# pylint: disable=unused-variable
19def check_rebase():
20    """ detect 'git rebase -x edk2-build.py master' testbuilds """
21    global rebase_prefix
22    global version_override
23    gitdir = '.git'
24
25    if os.path.isfile(gitdir):
26        with open(gitdir, 'r', encoding = 'utf-8') as f:
27            (unused, gitdir) = f.read().split()
28
29    if not os.path.exists(f'{gitdir}/rebase-merge/msgnum'):
30        return
31    with open(f'{gitdir}/rebase-merge/msgnum', 'r', encoding = 'utf-8') as f:
32        msgnum = int(f.read())
33    with open(f'{gitdir}/rebase-merge/end', 'r', encoding = 'utf-8') as f:
34        end = int(f.read())
35    with open(f'{gitdir}/rebase-merge/head-name', 'r', encoding = 'utf-8') as f:
36        head = f.read().strip().split('/')
37
38    rebase_prefix = f'[ {int(msgnum/2)} / {int(end/2)} - {head[-1]} ] '
39    if msgnum != end and not version_override:
40        # fixed version speeds up builds
41        version_override = "test-build-patch-series"
42
43def get_coredir(cfg):
44    if cfg.has_option('global', 'core'):
45        return os.path.abspath(cfg['global']['core'])
46    return os.getcwd()
47
48def get_version(cfg):
49    coredir = get_coredir(cfg)
50    if version_override:
51        version = version_override
52        print('')
53        print(f'### version [override]: {version}')
54        return version
55    if os.environ.get('RPM_PACKAGE_NAME'):
56        version = os.environ.get('RPM_PACKAGE_NAME')
57        version += '-' + os.environ.get('RPM_PACKAGE_VERSION')
58        version += '-' + os.environ.get('RPM_PACKAGE_RELEASE')
59        print('')
60        print(f'### version [rpmbuild]: {version}')
61        return version
62    if os.path.exists(coredir + '/.git'):
63        cmdline = [ 'git', 'describe', '--tags', '--abbrev=8',
64                    '--match=edk2-stable*' ]
65        result = subprocess.run(cmdline, cwd = coredir,
66                                stdout = subprocess.PIPE,
67                                check = True)
68        version = result.stdout.decode().strip()
69        print('')
70        print(f'### version [git]: {version}')
71        return version
72    return None
73
74def pcd_string(name, value):
75    return f'{name}=L{value}\\0'
76
77def pcd_version(cfg):
78    version = get_version(cfg)
79    if version is None:
80        return []
81    return [ '--pcd', pcd_string('PcdFirmwareVersionString', version) ]
82
83def pcd_release_date():
84    if release_date is None:
85        return []
86    return [ '--pcd', pcd_string('PcdFirmwareReleaseDateString', release_date) ]
87
88def build_message(line, line2 = None):
89    if os.environ.get('TERM') in [ 'xterm', 'xterm-256color' ]:
90        # setxterm  title
91        start  = '\x1b]2;'
92        end    = '\x07'
93        print(f'{start}{rebase_prefix}{line}{end}', end = '')
94
95    print('')
96    print('###')
97    print(f'### {rebase_prefix}{line}')
98    if line2:
99        print(f'### {line2}')
100    print('###', flush = True)
101
102def build_run(cmdline, name, section, silent = False):
103    print(cmdline, flush = True)
104    if silent:
105        print('### building in silent mode ...', flush = True)
106        result = subprocess.run(cmdline, check = False,
107                                stdout = subprocess.PIPE,
108                                stderr = subprocess.STDOUT)
109
110        logfile = f'{section}.log'
111        print(f'### writing log to {logfile} ...')
112        with open(logfile, 'wb') as f:
113            f.write(result.stdout)
114
115        if result.returncode:
116            print('### BUILD FAILURE')
117            print('### output')
118            print(result.stdout.decode())
119            print(f'### exit code: {result.returncode}')
120        else:
121            print('### OK')
122    else:
123        result = subprocess.run(cmdline, check = False)
124    if result.returncode:
125        print(f'ERROR: {cmdline[0]} exited with {result.returncode}'
126              f' while building {name}')
127        sys.exit(result.returncode)
128
129def build_copy(plat, tgt, dstdir, copy):
130    srcdir = f'Build/{plat}/{tgt}_GCC5'
131    names = copy.split()
132    srcfile = names[0]
133    if len(names) > 1:
134        dstfile = names[1]
135    else:
136        dstfile = os.path.basename(srcfile)
137    print(f'# copy: {srcdir} / {srcfile}  =>  {dstdir} / {dstfile}')
138
139    src = srcdir + '/' + srcfile
140    dst = dstdir + '/' + dstfile
141    os.makedirs(os.path.dirname(dst), exist_ok = True)
142    shutil.copy(src, dst)
143
144def pad_file(dstdir, pad):
145    args = pad.split()
146    if len(args) < 2:
147        raise RuntimeError(f'missing arg for pad ({args})')
148    name = args[0]
149    size = args[1]
150    cmdline = [
151        'truncate',
152        '--size', size,
153        dstdir + '/' + name,
154    ]
155    print(f'# padding: {dstdir} / {name}  =>  {size}')
156    subprocess.run(cmdline, check = True)
157
158# pylint: disable=too-many-branches
159def build_one(cfg, build, jobs = None, silent = False):
160    cmdline  = [ 'build' ]
161    cmdline += [ '-t', 'GCC5' ]
162    cmdline += [ '-p', cfg[build]['conf'] ]
163
164    if (cfg[build]['conf'].startswith('OvmfPkg/') or
165        cfg[build]['conf'].startswith('ArmVirtPkg/')):
166        cmdline += pcd_version(cfg)
167        cmdline += pcd_release_date()
168
169    if jobs:
170        cmdline += [ '-n', jobs ]
171    for arch in cfg[build]['arch'].split():
172        cmdline += [ '-a', arch ]
173    if 'opts' in cfg[build]:
174        for name in cfg[build]['opts'].split():
175            section = 'opts.' + name
176            for opt in cfg[section]:
177                cmdline += [ '-D', opt + '=' + cfg[section][opt] ]
178    if 'pcds' in cfg[build]:
179        for name in cfg[build]['pcds'].split():
180            section = 'pcds.' + name
181            for pcd in cfg[section]:
182                cmdline += [ '--pcd', pcd + '=' + cfg[section][pcd] ]
183    if 'tgts' in cfg[build]:
184        tgts = cfg[build]['tgts'].split()
185    else:
186        tgts = [ 'DEBUG' ]
187    for tgt in tgts:
188        desc = None
189        if 'desc' in cfg[build]:
190            desc = cfg[build]['desc']
191        build_message(f'building: {cfg[build]["conf"]} ({cfg[build]["arch"]}, {tgt})',
192                      f'description: {desc}')
193        build_run(cmdline + [ '-b', tgt ],
194                  cfg[build]['conf'],
195                  build + '.' + tgt,
196                  silent)
197
198        if 'plat' in cfg[build]:
199            # copy files
200            for cpy in cfg[build]:
201                if not cpy.startswith('cpy'):
202                    continue
203                build_copy(cfg[build]['plat'],
204                           tgt,
205                           cfg[build]['dest'],
206                           cfg[build][cpy])
207            # pad builds
208            for pad in cfg[build]:
209                if not pad.startswith('pad'):
210                    continue
211                pad_file(cfg[build]['dest'],
212                         cfg[build][pad])
213
214def build_basetools(silent = False):
215    build_message('building: BaseTools')
216    basedir = os.environ['EDK_TOOLS_PATH']
217    cmdline = [ 'make', '-C', basedir ]
218    build_run(cmdline, 'BaseTools', 'build.basetools', silent)
219
220def binary_exists(name):
221    for pdir in os.environ['PATH'].split(':'):
222        if os.path.exists(pdir + '/' + name):
223            return True
224    return False
225
226def prepare_env(cfg):
227    """ mimic Conf/BuildEnv.sh """
228    workspace = os.getcwd()
229    packages = [ workspace, ]
230    path = os.environ['PATH'].split(':')
231    dirs = [
232        'BaseTools/Bin/Linux-x86_64',
233        'BaseTools/BinWrappers/PosixLike'
234    ]
235
236    if cfg.has_option('global', 'pkgs'):
237        for pkgdir in cfg['global']['pkgs'].split():
238            packages.append(os.path.abspath(pkgdir))
239    coredir = get_coredir(cfg)
240    if coredir != workspace:
241        packages.append(coredir)
242
243    # add basetools to path
244    for pdir in dirs:
245        p = coredir + '/' + pdir
246        if not os.path.exists(p):
247            continue
248        if p in path:
249            continue
250        path.insert(0, p)
251
252    # run edksetup if needed
253    toolsdef = coredir + '/Conf/tools_def.txt'
254    if not os.path.exists(toolsdef):
255        os.makedirs(os.path.dirname(toolsdef), exist_ok = True)
256        build_message('running BaseTools/BuildEnv')
257        cmdline = [ 'bash', 'BaseTools/BuildEnv' ]
258        subprocess.run(cmdline, cwd = coredir, check = True)
259
260    # set variables
261    os.environ['PATH'] = ':'.join(path)
262    os.environ['PACKAGES_PATH'] = ':'.join(packages)
263    os.environ['WORKSPACE'] = workspace
264    os.environ['EDK_TOOLS_PATH'] = coredir + '/BaseTools'
265    os.environ['CONF_PATH'] = coredir + '/Conf'
266    os.environ['PYTHON_COMMAND'] = '/usr/bin/python3'
267    os.environ['PYTHONHASHSEED'] = '1'
268
269    # for cross builds
270    if binary_exists('arm-linux-gnu-gcc'):
271        os.environ['GCC5_ARM_PREFIX'] = 'arm-linux-gnu-'
272    if binary_exists('loongarch64-linux-gnu-gcc'):
273        os.environ['GCC5_LOONGARCH64_PREFIX'] = 'loongarch64-linux-gnu-'
274
275    hostarch = os.uname().machine
276    if binary_exists('aarch64-linux-gnu-gcc') and hostarch != 'aarch64':
277        os.environ['GCC5_AARCH64_PREFIX'] = 'aarch64-linux-gnu-'
278    if binary_exists('riscv64-linux-gnu-gcc') and hostarch != 'riscv64':
279        os.environ['GCC5_RISCV64_PREFIX'] = 'riscv64-linux-gnu-'
280    if binary_exists('x86_64-linux-gnu-gcc') and hostarch != 'x86_64':
281        os.environ['GCC5_IA32_PREFIX'] = 'x86_64-linux-gnu-'
282        os.environ['GCC5_X64_PREFIX'] = 'x86_64-linux-gnu-'
283        os.environ['GCC5_BIN'] = 'x86_64-linux-gnu-'
284
285def build_list(cfg):
286    for build in cfg.sections():
287        if not build.startswith('build.'):
288            continue
289        name = build.lstrip('build.')
290        desc = 'no description'
291        if 'desc' in cfg[build]:
292            desc = cfg[build]['desc']
293        print(f'# {name:20s} - {desc}')
294
295def main():
296    parser = argparse.ArgumentParser(prog = 'edk2-build',
297                                     description = 'edk2 build helper script')
298    parser.add_argument('-c', '--config', dest = 'configfile',
299                        type = str, default = '.edk2.builds', metavar = 'FILE',
300                        help = 'read configuration from FILE (default: .edk2.builds)')
301    parser.add_argument('-C', '--directory', dest = 'directory', type = str,
302                        help = 'change to DIR before building', metavar = 'DIR')
303    parser.add_argument('-j', '--jobs', dest = 'jobs', type = str,
304                        help = 'allow up to JOBS parallel build jobs',
305                        metavar = 'JOBS')
306    parser.add_argument('-m', '--match', dest = 'match', type = str,
307                        help = 'only run builds matching INCLUDE (substring)',
308                        metavar = 'INCLUDE')
309    parser.add_argument('-x', '--exclude', dest = 'exclude', type = str,
310                        help = 'skip builds matching EXCLUDE (substring)',
311                        metavar = 'EXCLUDE')
312    parser.add_argument('-l', '--list', dest = 'list',
313                        action = 'store_true', default = False,
314                        help = 'list build configs available')
315    parser.add_argument('--silent', dest = 'silent',
316                        action = 'store_true', default = False,
317                        help = 'write build output to logfiles, '
318                        'write to console only on errors')
319    parser.add_argument('--core', dest = 'core', type = str, metavar = 'DIR',
320                        help = 'location of the core edk2 repository '
321                        '(i.e. where BuildTools are located)')
322    parser.add_argument('--pkg', '--package', dest = 'pkgs',
323                        type = str, action = 'append', metavar = 'DIR',
324                        help = 'location(s) of additional packages '
325                        '(can be specified multiple times)')
326    parser.add_argument('--version-override', dest = 'version_override',
327                        type = str, metavar = 'VERSION',
328                        help = 'set firmware build version')
329    parser.add_argument('--release-date', dest = 'release_date',
330                        type = str, metavar = 'DATE',
331                        help = 'set firmware build release date (in MM/DD/YYYY format)')
332    options = parser.parse_args()
333
334    if options.directory:
335        os.chdir(options.directory)
336
337    if not os.path.exists(options.configfile):
338        print('config file "{options.configfile}" not found')
339        return 1
340
341    cfg = configparser.ConfigParser()
342    cfg.optionxform = str
343    cfg.read(options.configfile)
344
345    if options.list:
346        build_list(cfg)
347        return
348
349    if not cfg.has_section('global'):
350        cfg.add_section('global')
351    if options.core:
352        cfg.set('global', 'core', options.core)
353    if options.pkgs:
354        cfg.set('global', 'pkgs', ' '.join(options.pkgs))
355
356    global version_override
357    global release_date
358    check_rebase()
359    if options.version_override:
360        version_override = options.version_override
361    if options.release_date:
362        release_date = options.release_date
363
364    prepare_env(cfg)
365    build_basetools(options.silent)
366    for build in cfg.sections():
367        if not build.startswith('build.'):
368            continue
369        if options.match and options.match not in build:
370            print(f'# skipping "{build}" (not matching "{options.match}")')
371            continue
372        if options.exclude and options.exclude in build:
373            print(f'# skipping "{build}" (matching "{options.exclude}")')
374            continue
375        build_one(cfg, build, options.jobs, options.silent)
376
377    return 0
378
379if __name__ == '__main__':
380    sys.exit(main())
381