1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6 7import codecs 8import os 9import json 10import bb.compress.zstd 11import oe.path 12 13from glob import glob 14 15def packaged(pkg, d): 16 return os.access(get_subpkgedata_fn(pkg, d) + '.packaged', os.R_OK) 17 18def read_pkgdatafile(fn): 19 pkgdata = {} 20 21 def decode(str): 22 c = codecs.getdecoder("unicode_escape") 23 return c(str)[0] 24 25 if os.access(fn, os.R_OK): 26 import re 27 with open(fn, 'r') as f: 28 lines = f.readlines() 29 r = re.compile(r"(^.+?):\s+(.*)") 30 for l in lines: 31 m = r.match(l) 32 if m: 33 pkgdata[m.group(1)] = decode(m.group(2)) 34 35 return pkgdata 36 37def get_subpkgedata_fn(pkg, d): 38 return d.expand('${PKGDATA_DIR}/runtime/%s' % pkg) 39 40def has_subpkgdata(pkg, d): 41 return os.access(get_subpkgedata_fn(pkg, d), os.R_OK) 42 43def read_subpkgdata(pkg, d): 44 return read_pkgdatafile(get_subpkgedata_fn(pkg, d)) 45 46def has_pkgdata(pn, d): 47 fn = d.expand('${PKGDATA_DIR}/%s' % pn) 48 return os.access(fn, os.R_OK) 49 50def read_pkgdata(pn, d): 51 fn = d.expand('${PKGDATA_DIR}/%s' % pn) 52 return read_pkgdatafile(fn) 53 54# 55# Collapse FOO:pkg variables into FOO 56# 57def read_subpkgdata_dict(pkg, d): 58 ret = {} 59 subd = read_pkgdatafile(get_subpkgedata_fn(pkg, d)) 60 for var in subd: 61 newvar = var.replace(":" + pkg, "") 62 if newvar == var and var + ":" + pkg in subd: 63 continue 64 ret[newvar] = subd[var] 65 return ret 66 67def read_subpkgdata_extended(pkg, d): 68 import json 69 import bb.compress.zstd 70 71 fn = d.expand("${PKGDATA_DIR}/extended/%s.json.zstd" % pkg) 72 try: 73 num_threads = int(d.getVar("BB_NUMBER_THREADS")) 74 with bb.compress.zstd.open(fn, "rt", encoding="utf-8", num_threads=num_threads) as f: 75 return json.load(f) 76 except FileNotFoundError: 77 return None 78 79def _pkgmap(d): 80 """Return a dictionary mapping package to recipe name.""" 81 82 pkgdatadir = d.getVar("PKGDATA_DIR") 83 84 pkgmap = {} 85 try: 86 files = os.listdir(pkgdatadir) 87 except OSError: 88 bb.warn("No files in %s?" % pkgdatadir) 89 files = [] 90 91 for pn in [f for f in files if not os.path.isdir(os.path.join(pkgdatadir, f))]: 92 try: 93 pkgdata = read_pkgdatafile(os.path.join(pkgdatadir, pn)) 94 except OSError: 95 continue 96 97 packages = pkgdata.get("PACKAGES") or "" 98 for pkg in packages.split(): 99 pkgmap[pkg] = pn 100 101 return pkgmap 102 103def pkgmap(d): 104 """Return a dictionary mapping package to recipe name. 105 Cache the mapping in the metadata""" 106 107 pkgmap_data = d.getVar("__pkgmap_data", False) 108 if pkgmap_data is None: 109 pkgmap_data = _pkgmap(d) 110 d.setVar("__pkgmap_data", pkgmap_data) 111 112 return pkgmap_data 113 114def recipename(pkg, d): 115 """Return the recipe name for the given binary package name.""" 116 117 return pkgmap(d).get(pkg) 118 119def foreach_runtime_provider_pkgdata(d, rdep, include_rdep=False): 120 pkgdata_dir = d.getVar("PKGDATA_DIR") 121 possibles = set() 122 try: 123 possibles |= set(os.listdir("%s/runtime-rprovides/%s/" % (pkgdata_dir, rdep))) 124 except OSError: 125 pass 126 127 if include_rdep: 128 possibles.add(rdep) 129 130 for p in sorted(list(possibles)): 131 rdep_data = read_subpkgdata(p, d) 132 yield p, rdep_data 133 134def get_package_mapping(pkg, basepkg, d, depversions=None): 135 import oe.packagedata 136 137 data = oe.packagedata.read_subpkgdata(pkg, d) 138 key = "PKG:%s" % pkg 139 140 if key in data: 141 if bb.data.inherits_class('allarch', d) and bb.data.inherits_class('packagegroup', d) and pkg != data[key]: 142 bb.error("An allarch packagegroup shouldn't depend on packages which are dynamically renamed (%s to %s)" % (pkg, data[key])) 143 # Have to avoid undoing the write_extra_pkgs(global_variants...) 144 if bb.data.inherits_class('allarch', d) and not d.getVar('MULTILIB_VARIANTS') \ 145 and data[key] == basepkg: 146 return pkg 147 if depversions == []: 148 # Avoid returning a mapping if the renamed package rprovides its original name 149 rprovkey = "RPROVIDES:%s" % pkg 150 if rprovkey in data: 151 if pkg in bb.utils.explode_dep_versions2(data[rprovkey]): 152 bb.note("%s rprovides %s, not replacing the latter" % (data[key], pkg)) 153 return pkg 154 # Do map to rewritten package name 155 return data[key] 156 157 return pkg 158 159def get_package_additional_metadata(pkg_type, d): 160 base_key = "PACKAGE_ADD_METADATA" 161 for key in ("%s_%s" % (base_key, pkg_type.upper()), base_key): 162 if d.getVar(key, False) is None: 163 continue 164 d.setVarFlag(key, "type", "list") 165 if d.getVarFlag(key, "separator") is None: 166 d.setVarFlag(key, "separator", "\\n") 167 metadata_fields = [field.strip() for field in oe.data.typed_value(key, d)] 168 return "\n".join(metadata_fields).strip() 169 170def runtime_mapping_rename(varname, pkg, d): 171 #bb.note("%s before: %s" % (varname, d.getVar(varname))) 172 173 new_depends = {} 174 deps = bb.utils.explode_dep_versions2(d.getVar(varname) or "") 175 for depend, depversions in deps.items(): 176 new_depend = get_package_mapping(depend, pkg, d, depversions) 177 if depend != new_depend: 178 bb.note("package name mapping done: %s -> %s" % (depend, new_depend)) 179 new_depends[new_depend] = deps[depend] 180 181 d.setVar(varname, bb.utils.join_deps(new_depends, commasep=False)) 182 183 #bb.note("%s after: %s" % (varname, d.getVar(varname))) 184 185def emit_pkgdata(pkgfiles, d): 186 def process_postinst_on_target(pkg, mlprefix): 187 pkgval = d.getVar('PKG:%s' % pkg) 188 if pkgval is None: 189 pkgval = pkg 190 191 defer_fragment = """ 192if [ -n "$D" ]; then 193 $INTERCEPT_DIR/postinst_intercept delay_to_first_boot %s mlprefix=%s 194 exit 0 195fi 196""" % (pkgval, mlprefix) 197 198 postinst = d.getVar('pkg_postinst:%s' % pkg) 199 postinst_ontarget = d.getVar('pkg_postinst_ontarget:%s' % pkg) 200 201 if postinst_ontarget: 202 bb.debug(1, 'adding deferred pkg_postinst_ontarget() to pkg_postinst() for %s' % pkg) 203 if not postinst: 204 postinst = '#!/bin/sh\n' 205 postinst += defer_fragment 206 postinst += postinst_ontarget 207 d.setVar('pkg_postinst:%s' % pkg, postinst) 208 209 def add_set_e_to_scriptlets(pkg): 210 for scriptlet_name in ('pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm'): 211 scriptlet = d.getVar('%s:%s' % (scriptlet_name, pkg)) 212 if scriptlet: 213 scriptlet_split = scriptlet.split('\n') 214 if scriptlet_split[0].startswith("#!"): 215 scriptlet = scriptlet_split[0] + "\nset -e\n" + "\n".join(scriptlet_split[1:]) 216 else: 217 scriptlet = "set -e\n" + "\n".join(scriptlet_split[0:]) 218 d.setVar('%s:%s' % (scriptlet_name, pkg), scriptlet) 219 220 def write_if_exists(f, pkg, var): 221 def encode(str): 222 import codecs 223 c = codecs.getencoder("unicode_escape") 224 return c(str)[0].decode("latin1") 225 226 val = d.getVar('%s:%s' % (var, pkg)) 227 if val: 228 f.write('%s:%s: %s\n' % (var, pkg, encode(val))) 229 return val 230 val = d.getVar('%s' % (var)) 231 if val: 232 f.write('%s: %s\n' % (var, encode(val))) 233 return val 234 235 def write_extra_pkgs(variants, pn, packages, pkgdatadir): 236 for variant in variants: 237 with open("%s/%s-%s" % (pkgdatadir, variant, pn), 'w') as fd: 238 fd.write("PACKAGES: %s\n" % ' '.join( 239 map(lambda pkg: '%s-%s' % (variant, pkg), packages.split()))) 240 241 def write_extra_runtime_pkgs(variants, packages, pkgdatadir): 242 for variant in variants: 243 for pkg in packages.split(): 244 ml_pkg = "%s-%s" % (variant, pkg) 245 subdata_file = "%s/runtime/%s" % (pkgdatadir, ml_pkg) 246 with open(subdata_file, 'w') as fd: 247 fd.write("PKG:%s: %s" % (ml_pkg, pkg)) 248 249 packages = d.getVar('PACKAGES') 250 pkgdest = d.getVar('PKGDEST') 251 pkgdatadir = d.getVar('PKGDESTWORK') 252 253 data_file = pkgdatadir + d.expand("/${PN}") 254 with open(data_file, 'w') as fd: 255 fd.write("PACKAGES: %s\n" % packages) 256 257 pkgdebugsource = d.getVar("PKGDEBUGSOURCES") or [] 258 259 pn = d.getVar('PN') 260 global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split() 261 variants = (d.getVar('MULTILIB_VARIANTS') or "").split() 262 263 if bb.data.inherits_class('kernel', d) or bb.data.inherits_class('module-base', d): 264 write_extra_pkgs(variants, pn, packages, pkgdatadir) 265 266 if bb.data.inherits_class('allarch', d) and not variants \ 267 and not bb.data.inherits_class('packagegroup', d): 268 write_extra_pkgs(global_variants, pn, packages, pkgdatadir) 269 270 workdir = d.getVar('WORKDIR') 271 272 for pkg in packages.split(): 273 pkgval = d.getVar('PKG:%s' % pkg) 274 if pkgval is None: 275 pkgval = pkg 276 d.setVar('PKG:%s' % pkg, pkg) 277 278 extended_data = { 279 "files_info": {} 280 } 281 282 pkgdestpkg = os.path.join(pkgdest, pkg) 283 files = {} 284 files_extra = {} 285 total_size = 0 286 seen = set() 287 for f in pkgfiles[pkg]: 288 fpath = os.sep + os.path.relpath(f, pkgdestpkg) 289 290 fstat = os.lstat(f) 291 files[fpath] = fstat.st_size 292 293 extended_data["files_info"].setdefault(fpath, {}) 294 extended_data["files_info"][fpath]['size'] = fstat.st_size 295 296 if fstat.st_ino not in seen: 297 seen.add(fstat.st_ino) 298 total_size += fstat.st_size 299 300 if fpath in pkgdebugsource: 301 extended_data["files_info"][fpath]['debugsrc'] = pkgdebugsource[fpath] 302 del pkgdebugsource[fpath] 303 304 d.setVar('FILES_INFO:' + pkg , json.dumps(files, sort_keys=True)) 305 306 process_postinst_on_target(pkg, d.getVar("MLPREFIX")) 307 add_set_e_to_scriptlets(pkg) 308 309 subdata_file = pkgdatadir + "/runtime/%s" % pkg 310 with open(subdata_file, 'w') as sf: 311 for var in (d.getVar('PKGDATA_VARS') or "").split(): 312 val = write_if_exists(sf, pkg, var) 313 314 write_if_exists(sf, pkg, 'FILERPROVIDESFLIST') 315 for dfile in sorted((d.getVar('FILERPROVIDESFLIST:' + pkg) or "").split()): 316 write_if_exists(sf, pkg, 'FILERPROVIDES:' + dfile) 317 318 write_if_exists(sf, pkg, 'FILERDEPENDSFLIST') 319 for dfile in sorted((d.getVar('FILERDEPENDSFLIST:' + pkg) or "").split()): 320 write_if_exists(sf, pkg, 'FILERDEPENDS:' + dfile) 321 322 sf.write('%s:%s: %d\n' % ('PKGSIZE', pkg, total_size)) 323 324 subdata_extended_file = pkgdatadir + "/extended/%s.json.zstd" % pkg 325 num_threads = int(d.getVar("BB_NUMBER_THREADS")) 326 with bb.compress.zstd.open(subdata_extended_file, "wt", encoding="utf-8", num_threads=num_threads) as f: 327 json.dump(extended_data, f, sort_keys=True, separators=(",", ":")) 328 329 # Symlinks needed for rprovides lookup 330 rprov = d.getVar('RPROVIDES:%s' % pkg) or d.getVar('RPROVIDES') 331 if rprov: 332 for p in bb.utils.explode_deps(rprov): 333 subdata_sym = pkgdatadir + "/runtime-rprovides/%s/%s" % (p, pkg) 334 bb.utils.mkdirhier(os.path.dirname(subdata_sym)) 335 oe.path.relsymlink(subdata_file, subdata_sym, True) 336 337 allow_empty = d.getVar('ALLOW_EMPTY:%s' % pkg) 338 if not allow_empty: 339 allow_empty = d.getVar('ALLOW_EMPTY') 340 root = "%s/%s" % (pkgdest, pkg) 341 os.chdir(root) 342 g = glob('*') 343 if g or allow_empty == "1": 344 # Symlinks needed for reverse lookups (from the final package name) 345 subdata_sym = pkgdatadir + "/runtime-reverse/%s" % pkgval 346 oe.path.relsymlink(subdata_file, subdata_sym, True) 347 348 packagedfile = pkgdatadir + '/runtime/%s.packaged' % pkg 349 open(packagedfile, 'w').close() 350 351 if bb.data.inherits_class('kernel', d) or bb.data.inherits_class('module-base', d): 352 write_extra_runtime_pkgs(variants, packages, pkgdatadir) 353 354 if bb.data.inherits_class('allarch', d) and not variants \ 355 and not bb.data.inherits_class('packagegroup', d): 356 write_extra_runtime_pkgs(global_variants, packages, pkgdatadir) 357 358def mapping_rename_hook(d): 359 """ 360 Rewrite variables to account for package renaming in things 361 like debian.bbclass or manual PKG variable name changes 362 """ 363 pkg = d.getVar("PKG") 364 oe.packagedata.runtime_mapping_rename("RDEPENDS", pkg, d) 365 oe.packagedata.runtime_mapping_rename("RRECOMMENDS", pkg, d) 366 oe.packagedata.runtime_mapping_rename("RSUGGESTS", pkg, d) 367