1eb8dc403SDave Cobbley#!/usr/bin/env python3 2c342db35SBrad Bishop# 3*92b42cb3SPatrick Williams# Copyright OpenEmbedded Contributors 4*92b42cb3SPatrick Williams# 5c342db35SBrad Bishop# SPDX-License-Identifier: GPL-2.0-only 6c342db35SBrad Bishop# 7eb8dc403SDave Cobbley 8eb8dc403SDave Cobbleyimport sys, os, subprocess, re, shutil 9eb8dc403SDave Cobbley 109aee5003SAndrew Geisslerallowed = ( 11eb8dc403SDave Cobbley # type is supported by dash 12eb8dc403SDave Cobbley 'if type systemctl >/dev/null 2>/dev/null; then', 13eb8dc403SDave Cobbley 'if type systemd-tmpfiles >/dev/null 2>/dev/null; then', 14eb8dc403SDave Cobbley 'type update-rc.d >/dev/null 2>/dev/null; then', 15eb8dc403SDave Cobbley 'command -v', 16eb8dc403SDave Cobbley # HOSTNAME is set locally 17eb8dc403SDave Cobbley 'buildhistory_single_commit "$CMDLINE" "$HOSTNAME"', 18eb8dc403SDave Cobbley # False-positive, match is a grep not shell expression 19eb8dc403SDave Cobbley 'grep "^$groupname:[^:]*:[^:]*:\\([^,]*,\\)*$username\\(,[^,]*\\)*"', 20eb8dc403SDave Cobbley # TODO verify dash's '. script args' behaviour 21eb8dc403SDave Cobbley '. $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE' 22eb8dc403SDave Cobbley ) 23eb8dc403SDave Cobbley 249aee5003SAndrew Geisslerdef is_allowed(s): 259aee5003SAndrew Geissler for w in allowed: 26eb8dc403SDave Cobbley if w in s: 27eb8dc403SDave Cobbley return True 28eb8dc403SDave Cobbley return False 29eb8dc403SDave Cobbley 30eb8dc403SDave CobbleySCRIPT_LINENO_RE = re.compile(r' line (\d+) ') 31eb8dc403SDave CobbleyBASHISM_WARNING = re.compile(r'^(possible bashism in.*)$', re.MULTILINE) 32eb8dc403SDave Cobbley 33eb8dc403SDave Cobbleydef process(filename, function, lineno, script): 34eb8dc403SDave Cobbley import tempfile 35eb8dc403SDave Cobbley 36eb8dc403SDave Cobbley if not script.startswith("#!"): 37eb8dc403SDave Cobbley script = "#! /bin/sh\n" + script 38eb8dc403SDave Cobbley 39eb8dc403SDave Cobbley fn = tempfile.NamedTemporaryFile(mode="w+t") 40eb8dc403SDave Cobbley fn.write(script) 41eb8dc403SDave Cobbley fn.flush() 42eb8dc403SDave Cobbley 43eb8dc403SDave Cobbley try: 44eb8dc403SDave Cobbley subprocess.check_output(("checkbashisms.pl", fn.name), universal_newlines=True, stderr=subprocess.STDOUT) 45eb8dc403SDave Cobbley # No bashisms, so just return 46eb8dc403SDave Cobbley return 47eb8dc403SDave Cobbley except subprocess.CalledProcessError as e: 48eb8dc403SDave Cobbley # TODO check exit code is 1 49eb8dc403SDave Cobbley 50eb8dc403SDave Cobbley # Replace the temporary filename with the function and split it 51eb8dc403SDave Cobbley output = e.output.replace(fn.name, function) 52eb8dc403SDave Cobbley if not output or not output.startswith('possible bashism'): 53eb8dc403SDave Cobbley # Probably starts with or contains only warnings. Dump verbatim 549aee5003SAndrew Geissler # with one space indention. Can't do the splitting and allowed 55eb8dc403SDave Cobbley # checking below. 56eb8dc403SDave Cobbley return '\n'.join([filename, 57eb8dc403SDave Cobbley ' Unexpected output from checkbashisms.pl'] + 58eb8dc403SDave Cobbley [' ' + x for x in output.splitlines()]) 59eb8dc403SDave Cobbley 60eb8dc403SDave Cobbley # We know that the first line matches and that therefore the first 61eb8dc403SDave Cobbley # list entry will be empty - skip it. 62eb8dc403SDave Cobbley output = BASHISM_WARNING.split(output)[1:] 63eb8dc403SDave Cobbley # Turn the output into a single string like this: 64eb8dc403SDave Cobbley # /.../foobar.bb 65eb8dc403SDave Cobbley # possible bashism in updatercd_postrm line 2 (type): 66eb8dc403SDave Cobbley # if ${@use_updatercd(d)} && type update-rc.d >/dev/null 2>/dev/null; then 67eb8dc403SDave Cobbley # ... 68eb8dc403SDave Cobbley # ... 69eb8dc403SDave Cobbley result = [] 709aee5003SAndrew Geissler # Check the results against the allowed list 71eb8dc403SDave Cobbley for message, source in zip(output[0::2], output[1::2]): 72eb8dc403SDave Cobbley if not is_whitelisted(source): 73eb8dc403SDave Cobbley if lineno is not None: 74eb8dc403SDave Cobbley message = SCRIPT_LINENO_RE.sub(lambda m: ' line %d ' % (int(m.group(1)) + int(lineno) - 1), 75eb8dc403SDave Cobbley message) 76eb8dc403SDave Cobbley result.append(' ' + message.strip()) 77eb8dc403SDave Cobbley result.extend([' %s' % x for x in source.splitlines()]) 78eb8dc403SDave Cobbley if result: 79eb8dc403SDave Cobbley result.insert(0, filename) 80eb8dc403SDave Cobbley return '\n'.join(result) 81eb8dc403SDave Cobbley else: 82eb8dc403SDave Cobbley return None 83eb8dc403SDave Cobbley 84eb8dc403SDave Cobbleydef get_tinfoil(): 85eb8dc403SDave Cobbley scripts_path = os.path.dirname(os.path.realpath(__file__)) 86eb8dc403SDave Cobbley lib_path = scripts_path + '/lib' 87eb8dc403SDave Cobbley sys.path = sys.path + [lib_path] 88eb8dc403SDave Cobbley import scriptpath 89eb8dc403SDave Cobbley scriptpath.add_bitbake_lib_path() 90eb8dc403SDave Cobbley import bb.tinfoil 91eb8dc403SDave Cobbley tinfoil = bb.tinfoil.Tinfoil() 92eb8dc403SDave Cobbley tinfoil.prepare() 93eb8dc403SDave Cobbley # tinfoil.logger.setLevel(logging.WARNING) 94eb8dc403SDave Cobbley return tinfoil 95eb8dc403SDave Cobbley 96eb8dc403SDave Cobbleyif __name__=='__main__': 97eb8dc403SDave Cobbley import argparse, shutil 98eb8dc403SDave Cobbley 99eb8dc403SDave Cobbley parser = argparse.ArgumentParser(description='Bashim detector for shell fragments in recipes.') 100eb8dc403SDave Cobbley parser.add_argument("recipes", metavar="RECIPE", nargs="*", help="recipes to check (if not specified, all will be checked)") 101eb8dc403SDave Cobbley parser.add_argument("--verbose", default=False, action="store_true") 102eb8dc403SDave Cobbley args = parser.parse_args() 103eb8dc403SDave Cobbley 104eb8dc403SDave Cobbley if shutil.which("checkbashisms.pl") is None: 10595ac1b8dSAndrew Geissler print("Cannot find checkbashisms.pl on $PATH, get it from https://salsa.debian.org/debian/devscripts/raw/master/scripts/checkbashisms.pl") 106eb8dc403SDave Cobbley sys.exit(1) 107eb8dc403SDave Cobbley 108eb8dc403SDave Cobbley # The order of defining the worker function, 109eb8dc403SDave Cobbley # initializing the pool and connecting to the 110eb8dc403SDave Cobbley # bitbake server is crucial, don't change it. 111eb8dc403SDave Cobbley def func(item): 112eb8dc403SDave Cobbley (filename, key, lineno), script = item 113eb8dc403SDave Cobbley if args.verbose: 114eb8dc403SDave Cobbley print("Scanning %s:%s" % (filename, key)) 115eb8dc403SDave Cobbley return process(filename, key, lineno, script) 116eb8dc403SDave Cobbley 117eb8dc403SDave Cobbley import multiprocessing 118eb8dc403SDave Cobbley pool = multiprocessing.Pool() 119eb8dc403SDave Cobbley 120eb8dc403SDave Cobbley tinfoil = get_tinfoil() 121eb8dc403SDave Cobbley 122eb8dc403SDave Cobbley # This is only the default configuration and should iterate over 123eb8dc403SDave Cobbley # recipecaches to handle multiconfig environments 124eb8dc403SDave Cobbley pkg_pn = tinfoil.cooker.recipecaches[""].pkg_pn 125eb8dc403SDave Cobbley 126eb8dc403SDave Cobbley if args.recipes: 127eb8dc403SDave Cobbley initial_pns = args.recipes 128eb8dc403SDave Cobbley else: 129eb8dc403SDave Cobbley initial_pns = sorted(pkg_pn) 130eb8dc403SDave Cobbley 131eb8dc403SDave Cobbley pns = set() 132eb8dc403SDave Cobbley scripts = {} 133eb8dc403SDave Cobbley print("Generating scripts...") 134eb8dc403SDave Cobbley for pn in initial_pns: 135eb8dc403SDave Cobbley for fn in pkg_pn[pn]: 136eb8dc403SDave Cobbley # There's no point checking multiple BBCLASSEXTENDed variants of the same recipe 137eb8dc403SDave Cobbley # (at least in general - there is some risk that the variants contain different scripts) 138eb8dc403SDave Cobbley realfn, _, _ = bb.cache.virtualfn2realfn(fn) 139eb8dc403SDave Cobbley if realfn not in pns: 140eb8dc403SDave Cobbley pns.add(realfn) 141eb8dc403SDave Cobbley data = tinfoil.parse_recipe_file(realfn) 142eb8dc403SDave Cobbley for key in data.keys(): 143eb8dc403SDave Cobbley if data.getVarFlag(key, "func") and not data.getVarFlag(key, "python"): 144eb8dc403SDave Cobbley script = data.getVar(key, False) 145eb8dc403SDave Cobbley if script: 146eb8dc403SDave Cobbley filename = data.getVarFlag(key, "filename") 147eb8dc403SDave Cobbley lineno = data.getVarFlag(key, "lineno") 148eb8dc403SDave Cobbley # There's no point in checking a function multiple 149eb8dc403SDave Cobbley # times just because different recipes include it. 150eb8dc403SDave Cobbley # We identify unique scripts by file, name, and (just in case) 151eb8dc403SDave Cobbley # line number. 152eb8dc403SDave Cobbley attributes = (filename or realfn, key, lineno) 153eb8dc403SDave Cobbley scripts.setdefault(attributes, script) 154eb8dc403SDave Cobbley 155eb8dc403SDave Cobbley 156eb8dc403SDave Cobbley print("Scanning scripts...\n") 157eb8dc403SDave Cobbley for result in pool.imap(func, scripts.items()): 158eb8dc403SDave Cobbley if result: 159eb8dc403SDave Cobbley print(result) 160eb8dc403SDave Cobbley tinfoil.shutdown() 161