1#!/usr/bin/env python3 2# 3# Conversion script to add new override syntax to existing bitbake metadata 4# 5# Copyright (C) 2021 Richard Purdie 6# 7# SPDX-License-Identifier: GPL-2.0-only 8# 9 10# 11# To use this script on a new layer you need to list the overrides the 12# layer is known to use in the list below. 13# 14# Known constraint: Matching is 'loose' and in particular will find variable 15# and function names with "_append" and "_remove" in them. Those need to be 16# filtered out manually or in the skip list below. 17# 18 19import re 20import os 21import sys 22import tempfile 23import shutil 24import mimetypes 25 26if len(sys.argv) < 2: 27 print("Please specify a directory to run the conversion script against.") 28 sys.exit(1) 29 30# List of strings to treat as overrides 31vars = ["append", "prepend", "remove"] 32vars = vars + ["qemuarm", "qemux86", "qemumips", "qemuppc", "qemuriscv", "qemuall"] 33vars = vars + ["genericx86", "edgerouter", "beaglebone-yocto"] 34vars = vars + ["armeb", "arm", "armv5", "armv6", "armv4", "powerpc64", "aarch64", "riscv32", "riscv64", "x86", "mips64", "powerpc"] 35vars = vars + ["mipsarch", "x86-x32", "mips16e", "microblaze", "e5500-64b", "mipsisa32", "mipsisa64"] 36vars = vars + ["class-native", "class-target", "class-cross-canadian", "class-cross", "class-devupstream"] 37vars = vars + ["tune-", "pn-", "forcevariable"] 38vars = vars + ["libc-musl", "libc-glibc", "libc-newlib","libc-baremetal"] 39vars = vars + ["task-configure", "task-compile", "task-install", "task-clean", "task-image-qa", "task-rm_work", "task-image-complete", "task-populate-sdk"] 40vars = vars + ["toolchain-clang", "mydistro", "nios2", "sdkmingw32", "overrideone", "overridetwo"] 41vars = vars + ["linux-gnux32", "linux-muslx32", "linux-gnun32", "mingw32", "poky", "darwin", "linuxstdbase"] 42vars = vars + ["linux-gnueabi", "eabi"] 43vars = vars + ["virtclass-multilib", "virtclass-mcextend"] 44 45# List of strings to treat as overrides but only with whitespace following or another override (more restricted matching). 46# Handles issues with arc matching arch. 47shortvars = ["arc", "mips", "mipsel", "sh4"] 48 49# Variables which take packagenames as an override 50packagevars = ["FILES", "RDEPENDS", "RRECOMMENDS", "SUMMARY", "DESCRIPTION", "RSUGGESTS", "RPROVIDES", "RCONFLICTS", "PKG", "ALLOW_EMPTY", 51 "pkg_postrm", "pkg_postinst_ontarget", "pkg_postinst", "INITSCRIPT_NAME", "INITSCRIPT_PARAMS", "DEBIAN_NOAUTONAME", "ALTERNATIVE", 52 "PKGE", "PKGV", "PKGR", "USERADD_PARAM", "GROUPADD_PARAM", "CONFFILES", "SYSTEMD_SERVICE", "LICENSE", "SECTION", "pkg_preinst", 53 "pkg_prerm", "RREPLACES", "GROUPMEMS_PARAM", "SYSTEMD_AUTO_ENABLE", "SKIP_FILEDEPS", "PRIVATE_LIBS", "PACKAGE_ADD_METADATA", 54 "INSANE_SKIP", "DEBIANNAME", "SYSTEMD_SERVICE_ESCAPED"] 55 56# Expressions to skip if encountered, these are not overrides 57skips = ["parser_append", "recipe_to_append", "extra_append", "to_remove", "show_appends", "applied_appends", "file_appends", "handle_remove"] 58skips = skips + ["expanded_removes", "color_remove", "test_remove", "empty_remove", "toaster_prepend", "num_removed", "licfiles_append", "_write_append"] 59skips = skips + ["no_report_remove", "test_prepend", "test_append", "multiple_append", "test_remove", "shallow_remove", "do_remove_layer", "first_append"] 60skips = skips + ["parser_remove", "to_append", "no_remove", "bblayers_add_remove", "bblayers_remove", "apply_append", "is_x86", "base_dep_prepend"] 61skips = skips + ["autotools_dep_prepend", "go_map_arm", "alt_remove_links", "systemd_append_file", "file_append", "process_file_darwin"] 62skips = skips + ["run_loaddata_poky", "determine_if_poky_env", "do_populate_poky_src", "libc_cv_include_x86_isa_level", "test_rpm_remove", "do_install_armmultilib"] 63skips = skips + ["get_appends_for_files", "test_doubleref_remove", "test_bitbakelayers_add_remove", "elf32_x86_64", "colour_remove", "revmap_remove"] 64skips = skips + ["test_rpm_remove", "test_bitbakelayers_add_remove", "recipe_append_file", "log_data_removed", "recipe_append", "systemd_machine_unit_append"] 65skips = skips + ["recipetool_append", "changetype_remove", "try_appendfile_wc", "test_qemux86_directdisk", "test_layer_appends", "tgz_removed"] 66 67imagevars = ["IMAGE_CMD", "EXTRA_IMAGECMD", "IMAGE_TYPEDEP", "CONVERSION_CMD", "COMPRESS_CMD"] 68packagevars = packagevars + imagevars 69 70vars_re = {} 71for exp in vars: 72 vars_re[exp] = (re.compile('((^|[#\'"\s\-\+])[A-Za-z0-9_\-:${}\.]+)_' + exp), r"\1:" + exp) 73 74shortvars_re = {} 75for exp in shortvars: 76 shortvars_re[exp] = (re.compile('((^|[#\'"\s\-\+])[A-Za-z0-9_\-:${}\.]+)_' + exp + '([\(\'"\s:])'), r"\1:" + exp + r"\3") 77 78package_re = {} 79for exp in packagevars: 80 package_re[exp] = (re.compile('(^|[#\'"\s\-\+]+)' + exp + '_' + '([$a-z"\'\s%\[<{\\\*].)'), r"\1" + exp + r":\2") 81 82# Other substitutions to make 83subs = { 84 'r = re.compile("([^:]+):\s*(.*)")' : 'r = re.compile("(^.+?):\s+(.*)")', 85 "val = d.getVar('%s_%s' % (var, pkg))" : "val = d.getVar('%s:%s' % (var, pkg))", 86 "f.write('%s_%s: %s\\n' % (var, pkg, encode(val)))" : "f.write('%s:%s: %s\\n' % (var, pkg, encode(val)))", 87 "d.getVar('%s_%s' % (scriptlet_name, pkg))" : "d.getVar('%s:%s' % (scriptlet_name, pkg))", 88 'ret.append(v + "_" + p)' : 'ret.append(v + ":" + p)', 89} 90 91def processfile(fn): 92 print("processing file '%s'" % fn) 93 try: 94 fh, abs_path = tempfile.mkstemp() 95 with os.fdopen(fh, 'w') as new_file: 96 with open(fn, "r") as old_file: 97 for line in old_file: 98 skip = False 99 for s in skips: 100 if s in line: 101 skip = True 102 if "ptest_append" in line or "ptest_remove" in line or "ptest_prepend" in line: 103 skip = False 104 for sub in subs: 105 if sub in line: 106 line = line.replace(sub, subs[sub]) 107 skip = True 108 if not skip: 109 for pvar in packagevars: 110 line = package_re[pvar][0].sub(package_re[pvar][1], line) 111 for var in vars: 112 line = vars_re[var][0].sub(vars_re[var][1], line) 113 for shortvar in shortvars: 114 line = shortvars_re[shortvar][0].sub(shortvars_re[shortvar][1], line) 115 if "pkg_postinst:ontarget" in line: 116 line = line.replace("pkg_postinst:ontarget", "pkg_postinst_ontarget") 117 new_file.write(line) 118 shutil.copymode(fn, abs_path) 119 os.remove(fn) 120 shutil.move(abs_path, fn) 121 except UnicodeDecodeError: 122 pass 123 124ourname = os.path.basename(sys.argv[0]) 125ourversion = "0.9.3" 126 127if os.path.isfile(sys.argv[1]): 128 processfile(sys.argv[1]) 129 sys.exit(0) 130 131for targetdir in sys.argv[1:]: 132 print("processing directory '%s'" % targetdir) 133 for root, dirs, files in os.walk(targetdir): 134 for name in files: 135 if name == ourname: 136 continue 137 fn = os.path.join(root, name) 138 if os.path.islink(fn): 139 continue 140 if "/.git/" in fn or fn.endswith(".html") or fn.endswith(".patch") or fn.endswith(".m4") or fn.endswith(".diff"): 141 continue 142 processfile(fn) 143 144print("All files processed with version %s" % ourversion) 145