xref: /openbmc/qemu/roms/edk2-build.py (revision 5f88dd43) 
1#!/usr/bin/python3
2"""
3build helper script for edk2, see
4https://gitlab.com/kraxel/edk2-build-config
5
6"""
7import os
8import sys
9import time
10import shutil
11import argparse
12import subprocess
13import configparser
14
15rebase_prefix    = ""
16version_override = None
17release_date     = None
18
19# pylint: disable=unused-variable
20def check_rebase():
21    """ detect 'git rebase -x edk2-build.py master' testbuilds """
22    global rebase_prefix
23    global version_override
24    gitdir = '.git'
25
26    if os.path.isfile(gitdir):
27        with open(gitdir, 'r', encoding = 'utf-8') as f:
28            (unused, gitdir) = f.read().split()
29
30    if not os.path.exists(f'{gitdir}/rebase-merge/msgnum'):
31        return
32    with open(f'{gitdir}/rebase-merge/msgnum', 'r', encoding = 'utf-8') as f:
33        msgnum = int(f.read())
34    with open(f'{gitdir}/rebase-merge/end', 'r', encoding = 'utf-8') as f:
35        end = int(f.read())
36    with open(f'{gitdir}/rebase-merge/head-name', 'r', encoding = 'utf-8') as f:
37        head = f.read().strip().split('/')
38
39    rebase_prefix = f'[ {int(msgnum/2)} / {int(end/2)} - {head[-1]} ] '
40    if msgnum != end and not version_override:
41        # fixed version speeds up builds
42        version_override = "test-build-patch-series"
43
44def get_coredir(cfg):
45    if cfg.has_option('global', 'core'):
46        return os.path.abspath(cfg['global']['core'])
47    return os.getcwd()
48
49def get_toolchain(cfg, build):
50    if cfg.has_option(build, 'tool'):
51        return cfg[build]['tool']
52    if cfg.has_option('global', 'tool'):
53        return cfg['global']['tool']
54    return 'GCC5'
55
56def get_version(cfg, silent = False):
57    coredir = get_coredir(cfg)
58    if version_override:
59        version = version_override
60        if not silent:
61            print('')
62            print(f'### version [override]: {version}')
63        return version
64    if os.environ.get('RPM_PACKAGE_NAME'):
65        version = os.environ.get('RPM_PACKAGE_NAME')
66        version += '-' + os.environ.get('RPM_PACKAGE_VERSION')
67        version += '-' + os.environ.get('RPM_PACKAGE_RELEASE')
68        if not silent:
69            print('')
70            print(f'### version [rpmbuild]: {version}')
71        return version
72    if os.path.exists(coredir + '/.git'):
73        cmdline = [ 'git', 'describe', '--tags', '--abbrev=8',
74                    '--match=edk2-stable*' ]
75        result = subprocess.run(cmdline, cwd = coredir,
76                                stdout = subprocess.PIPE,
77                                check = True)
78        version = result.stdout.decode().strip()
79        if not silent:
80            print('')
81            print(f'### version [git]: {version}')
82        return version
83    return None
84
85def pcd_string(name, value):
86    return f'{name}=L{value}\\0'
87
88def pcd_version(cfg, silent = False):
89    version = get_version(cfg, silent)
90    if version is None:
91        return []
92    return [ '--pcd', pcd_string('PcdFirmwareVersionString', version) ]
93
94def pcd_release_date():
95    if release_date is None:
96        return []
97    return [ '--pcd', pcd_string('PcdFirmwareReleaseDateString', release_date) ]
98
99def build_message(line, line2 = None, silent = False):
100    if os.environ.get('TERM') in [ 'xterm', 'xterm-256color' ]:
101        # setxterm  title
102        start  = '\x1b]2;'
103        end    = '\x07'
104        print(f'{start}{rebase_prefix}{line}{end}', end = '')
105
106    if silent:
107        print(f'### {rebase_prefix}{line}', flush = True)
108    else:
109        print('')
110        print('###')
111        print(f'### {rebase_prefix}{line}')
112        if line2:
113            print(f'### {line2}')
114        print('###', flush = True)
115
116def build_run(cmdline, name, section, silent = False, nologs = False):
117    if silent:
118        logfile = f'{section}.log'
119        if nologs:
120            print(f'### building in silent mode [no log] ...', flush = True)
121        else:
122            print(f'### building in silent mode [{logfile}] ...', flush = True)
123        start = time.time()
124        result = subprocess.run(cmdline, check = False,
125                                stdout = subprocess.PIPE,
126                                stderr = subprocess.STDOUT)
127        if not nologs:
128            with open(logfile, 'wb') as f:
129                f.write(result.stdout)
130
131        if result.returncode:
132            print('### BUILD FAILURE')
133            print('### cmdline')
134            print(cmdline)
135            print('### output')
136            print(result.stdout.decode())
137            print(f'### exit code: {result.returncode}')
138        else:
139            secs = int(time.time() - start)
140            print(f'### OK ({int(secs/60)}:{secs%60:02d})')
141    else:
142        print(cmdline, flush = True)
143        result = subprocess.run(cmdline, check = False)
144    if result.returncode:
145        print(f'ERROR: {cmdline[0]} exited with {result.returncode}'
146              f' while building {name}')
147        sys.exit(result.returncode)
148
149def build_copy(plat, tgt, toolchain, dstdir, copy):
150    srcdir = f'Build/{plat}/{tgt}_{toolchain}'
151    names = copy.split()
152    srcfile = names[0]
153    if len(names) > 1:
154        dstfile = names[1]
155    else:
156        dstfile = os.path.basename(srcfile)
157    print(f'# copy: {srcdir} / {srcfile}  =>  {dstdir} / {dstfile}')
158
159    src = srcdir + '/' + srcfile
160    dst = dstdir + '/' + dstfile
161    os.makedirs(os.path.dirname(dst), exist_ok = True)
162    shutil.copy(src, dst)
163
164def pad_file(dstdir, pad):
165    args = pad.split()
166    if len(args) < 2:
167        raise RuntimeError(f'missing arg for pad ({args})')
168    name = args[0]
169    size = args[1]
170    cmdline = [
171        'truncate',
172        '--size', size,
173        dstdir + '/' + name,
174    ]
175    print(f'# padding: {dstdir} / {name}  =>  {size}')
176    subprocess.run(cmdline, check = True)
177
178# pylint: disable=too-many-branches
179def build_one(cfg, build, jobs = None, silent = False, nologs = False):
180    b = cfg[build]
181
182    cmdline  = [ 'build' ]
183    cmdline += [ '-t', get_toolchain(cfg, build) ]
184    cmdline += [ '-p', b['conf'] ]
185
186    if (b['conf'].startswith('OvmfPkg/') or
187        b['conf'].startswith('ArmVirtPkg/')):
188        cmdline += pcd_version(cfg, silent)
189        cmdline += pcd_release_date()
190
191    if jobs:
192        cmdline += [ '-n', jobs ]
193    for arch in b['arch'].split():
194        cmdline += [ '-a', arch ]
195    if 'opts' in b:
196        for name in b['opts'].split():
197            section = 'opts.' + name
198            for opt in cfg[section]:
199                cmdline += [ '-D', opt + '=' + cfg[section][opt] ]
200    if 'pcds' in b:
201        for name in b['pcds'].split():
202            section = 'pcds.' + name
203            for pcd in cfg[section]:
204                cmdline += [ '--pcd', pcd + '=' + cfg[section][pcd] ]
205    if 'tgts' in b:
206        tgts = b['tgts'].split()
207    else:
208        tgts = [ 'DEBUG' ]
209    for tgt in tgts:
210        desc = None
211        if 'desc' in b:
212            desc = b['desc']
213        build_message(f'building: {b["conf"]} ({b["arch"]}, {tgt})',
214                      f'description: {desc}',
215                      silent = silent)
216        build_run(cmdline + [ '-b', tgt ],
217                  b['conf'],
218                  build + '.' + tgt,
219                  silent,
220                  nologs)
221
222        if 'plat' in b:
223            # copy files
224            for cpy in b:
225                if not cpy.startswith('cpy'):
226                    continue
227                build_copy(b['plat'], tgt,
228                           get_toolchain(cfg, build),
229                           b['dest'], b[cpy])
230            # pad builds
231            for pad in b:
232                if not pad.startswith('pad'):
233                    continue
234                pad_file(b['dest'], b[pad])
235
236def build_basetools(silent = False, nologs = False):
237    build_message('building: BaseTools', silent = silent)
238    basedir = os.environ['EDK_TOOLS_PATH']
239    cmdline = [ 'make', '-C', basedir ]
240    build_run(cmdline, 'BaseTools', 'build.basetools', silent, nologs)
241
242def binary_exists(name):
243    for pdir in os.environ['PATH'].split(':'):
244        if os.path.exists(pdir + '/' + name):
245            return True
246    return False
247
248def prepare_env(cfg, silent = False):
249    """ mimic Conf/BuildEnv.sh """
250    workspace = os.getcwd()
251    packages = [ workspace, ]
252    path = os.environ['PATH'].split(':')
253    dirs = [
254        'BaseTools/Bin/Linux-x86_64',
255        'BaseTools/BinWrappers/PosixLike'
256    ]
257
258    if cfg.has_option('global', 'pkgs'):
259        for pkgdir in cfg['global']['pkgs'].split():
260            packages.append(os.path.abspath(pkgdir))
261    coredir = get_coredir(cfg)
262    if coredir != workspace:
263        packages.append(coredir)
264
265    # add basetools to path
266    for pdir in dirs:
267        p = coredir + '/' + pdir
268        if not os.path.exists(p):
269            continue
270        if p in path:
271            continue
272        path.insert(0, p)
273
274    # run edksetup if needed
275    toolsdef = coredir + '/Conf/tools_def.txt'
276    if not os.path.exists(toolsdef):
277        os.makedirs(os.path.dirname(toolsdef), exist_ok = True)
278        build_message('running BaseTools/BuildEnv', silent = silent)
279        cmdline = [ 'bash', 'BaseTools/BuildEnv' ]
280        subprocess.run(cmdline, cwd = coredir, check = True)
281
282    # set variables
283    os.environ['PATH'] = ':'.join(path)
284    os.environ['PACKAGES_PATH'] = ':'.join(packages)
285    os.environ['WORKSPACE'] = workspace
286    os.environ['EDK_TOOLS_PATH'] = coredir + '/BaseTools'
287    os.environ['CONF_PATH'] = coredir + '/Conf'
288    os.environ['PYTHON_COMMAND'] = '/usr/bin/python3'
289    os.environ['PYTHONHASHSEED'] = '1'
290
291    # for cross builds
292    if binary_exists('arm-linux-gnueabi-gcc'):
293        # ubuntu
294        os.environ['GCC5_ARM_PREFIX'] = 'arm-linux-gnueabi-'
295        os.environ['GCC_ARM_PREFIX'] = 'arm-linux-gnueabi-'
296    elif binary_exists('arm-linux-gnu-gcc'):
297        # fedora
298        os.environ['GCC5_ARM_PREFIX'] = 'arm-linux-gnu-'
299        os.environ['GCC_ARM_PREFIX'] = 'arm-linux-gnu-'
300    if binary_exists('loongarch64-linux-gnu-gcc'):
301        os.environ['GCC5_LOONGARCH64_PREFIX'] = 'loongarch64-linux-gnu-'
302        os.environ['GCC_LOONGARCH64_PREFIX'] = 'loongarch64-linux-gnu-'
303
304    hostarch = os.uname().machine
305    if binary_exists('aarch64-linux-gnu-gcc') and hostarch != 'aarch64':
306        os.environ['GCC5_AARCH64_PREFIX'] = 'aarch64-linux-gnu-'
307        os.environ['GCC_AARCH64_PREFIX'] = 'aarch64-linux-gnu-'
308    if binary_exists('riscv64-linux-gnu-gcc') and hostarch != 'riscv64':
309        os.environ['GCC5_RISCV64_PREFIX'] = 'riscv64-linux-gnu-'
310        os.environ['GCC_RISCV64_PREFIX'] = 'riscv64-linux-gnu-'
311    if binary_exists('x86_64-linux-gnu-gcc') and hostarch != 'x86_64':
312        os.environ['GCC5_IA32_PREFIX'] = 'x86_64-linux-gnu-'
313        os.environ['GCC5_X64_PREFIX'] = 'x86_64-linux-gnu-'
314        os.environ['GCC5_BIN'] = 'x86_64-linux-gnu-'
315        os.environ['GCC_IA32_PREFIX'] = 'x86_64-linux-gnu-'
316        os.environ['GCC_X64_PREFIX'] = 'x86_64-linux-gnu-'
317        os.environ['GCC_BIN'] = 'x86_64-linux-gnu-'
318
319def build_list(cfg):
320    for build in cfg.sections():
321        if not build.startswith('build.'):
322            continue
323        name = build.lstrip('build.')
324        desc = 'no description'
325        if 'desc' in cfg[build]:
326            desc = cfg[build]['desc']
327        print(f'# {name:20s} - {desc}')
328
329def main():
330    parser = argparse.ArgumentParser(prog = 'edk2-build',
331                                     description = 'edk2 build helper script')
332    parser.add_argument('-c', '--config', dest = 'configfile',
333                        type = str, default = '.edk2.builds', metavar = 'FILE',
334                        help = 'read configuration from FILE (default: .edk2.builds)')
335    parser.add_argument('-C', '--directory', dest = 'directory', type = str,
336                        help = 'change to DIR before building', metavar = 'DIR')
337    parser.add_argument('-j', '--jobs', dest = 'jobs', type = str,
338                        help = 'allow up to JOBS parallel build jobs',
339                        metavar = 'JOBS')
340    parser.add_argument('-m', '--match', dest = 'match',
341                        type = str, action = 'append',
342                        help = 'only run builds matching INCLUDE (substring)',
343                        metavar = 'INCLUDE')
344    parser.add_argument('-x', '--exclude', dest = 'exclude',
345                        type = str, action = 'append',
346                        help = 'skip builds matching EXCLUDE (substring)',
347                        metavar = 'EXCLUDE')
348    parser.add_argument('-l', '--list', dest = 'list',
349                        action = 'store_true', default = False,
350                        help = 'list build configs available')
351    parser.add_argument('--silent', dest = 'silent',
352                        action = 'store_true', default = False,
353                        help = 'write build output to logfiles, '
354                        'write to console only on errors')
355    parser.add_argument('--no-logs', dest = 'nologs',
356                        action = 'store_true', default = False,
357                        help = 'do not write build log files (with --silent)')
358    parser.add_argument('--core', dest = 'core', type = str, metavar = 'DIR',
359                        help = 'location of the core edk2 repository '
360                        '(i.e. where BuildTools are located)')
361    parser.add_argument('--pkg', '--package', dest = 'pkgs',
362                        type = str, action = 'append', metavar = 'DIR',
363                        help = 'location(s) of additional packages '
364                        '(can be specified multiple times)')
365    parser.add_argument('-t', '--toolchain', dest = 'toolchain',
366                        type = str, metavar = 'NAME',
367                        help = 'tool chain to be used to build edk2')
368    parser.add_argument('--version-override', dest = 'version_override',
369                        type = str, metavar = 'VERSION',
370                        help = 'set firmware build version')
371    parser.add_argument('--release-date', dest = 'release_date',
372                        type = str, metavar = 'DATE',
373                        help = 'set firmware build release date (in MM/DD/YYYY format)')
374    options = parser.parse_args()
375
376    if options.directory:
377        os.chdir(options.directory)
378
379    if not os.path.exists(options.configfile):
380        print(f'config file "{options.configfile}" not found')
381        return 1
382
383    cfg = configparser.ConfigParser()
384    cfg.optionxform = str
385    cfg.read(options.configfile)
386
387    if options.list:
388        build_list(cfg)
389        return 0
390
391    if not cfg.has_section('global'):
392        cfg.add_section('global')
393    if options.core:
394        cfg.set('global', 'core', options.core)
395    if options.pkgs:
396        cfg.set('global', 'pkgs', ' '.join(options.pkgs))
397    if options.toolchain:
398        cfg.set('global', 'tool', options.toolchain)
399
400    global version_override
401    global release_date
402    check_rebase()
403    if options.version_override:
404        version_override = options.version_override
405    if options.release_date:
406        release_date = options.release_date
407
408    prepare_env(cfg, options.silent)
409    build_basetools(options.silent, options.nologs)
410    for build in cfg.sections():
411        if not build.startswith('build.'):
412            continue
413        if options.match:
414            matching = False
415            for item in options.match:
416                if item in build:
417                    matching = True
418            if not matching:
419                print(f'# skipping "{build}" (not matching "{"|".join(options.match)}")')
420                continue
421        if options.exclude:
422            exclude = False
423            for item in options.exclude:
424                if item in build:
425                    print(f'# skipping "{build}" (matching "{item}")')
426                    exclude = True
427            if exclude:
428                continue
429        build_one(cfg, build, options.jobs, options.silent, options.nologs)
430
431    return 0
432
433if __name__ == '__main__':
434    sys.exit(main())
435