1# 2# Records history of build output in order to detect regressions 3# 4# Based in part on testlab.bbclass and packagehistory.bbclass 5# 6# Copyright (C) 2011-2016 Intel Corporation 7# Copyright (C) 2007-2011 Koen Kooi <koen@openembedded.org> 8# 9# SPDX-License-Identifier: MIT 10# 11 12IMAGE_CLASSES += "image-artifact-names" 13 14BUILDHISTORY_FEATURES ?= "image package sdk" 15BUILDHISTORY_DIR ?= "${TOPDIR}/buildhistory" 16BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}" 17BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}" 18 19# Setting this to non-empty will remove the old content of the buildhistory as part of 20# the current bitbake invocation and replace it with information about what was built 21# during the build. 22# 23# This is meant to be used in continuous integration (CI) systems when invoking bitbake 24# for full world builds. The effect in that case is that information about packages 25# that no longer get build also gets removed from the buildhistory, which is not 26# the case otherwise. 27# 28# The advantage over manually cleaning the buildhistory outside of bitbake is that 29# the "version-going-backwards" check still works. When relying on that, be careful 30# about failed world builds: they will lead to incomplete information in the 31# buildhistory because information about packages that could not be built will 32# also get removed. A CI system should handle that by discarding the buildhistory 33# of failed builds. 34# 35# The expected usage is via auto.conf, but passing via the command line also works 36# with: BB_ENV_PASSTHROUGH_ADDITIONS=BUILDHISTORY_RESET BUILDHISTORY_RESET=1 37BUILDHISTORY_RESET ?= "" 38 39BUILDHISTORY_OLD_DIR = "${BUILDHISTORY_DIR}/${@ "old" if "${BUILDHISTORY_RESET}" else ""}" 40BUILDHISTORY_OLD_DIR_PACKAGE = "${BUILDHISTORY_OLD_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}" 41BUILDHISTORY_DIR_SDK = "${BUILDHISTORY_DIR}/sdk/${SDK_NAME}${SDK_EXT}/${IMAGE_BASENAME}" 42BUILDHISTORY_IMAGE_FILES ?= "/etc/passwd /etc/group" 43BUILDHISTORY_SDK_FILES ?= "conf/local.conf conf/bblayers.conf conf/auto.conf conf/locked-sigs.inc conf/devtool.conf" 44BUILDHISTORY_COMMIT ?= "1" 45BUILDHISTORY_COMMIT_AUTHOR ?= "buildhistory <buildhistory@${DISTRO}>" 46BUILDHISTORY_PUSH_REPO ?= "" 47BUILDHISTORY_TAG ?= "build" 48BUILDHISTORY_PATH_PREFIX_STRIP ?= "" 49 50SSTATEPOSTINSTFUNCS:append = " buildhistory_emit_pkghistory" 51# We want to avoid influencing the signatures of sstate tasks - first the function itself: 52sstate_install[vardepsexclude] += "buildhistory_emit_pkghistory" 53# then the value added to SSTATEPOSTINSTFUNCS: 54SSTATEPOSTINSTFUNCS[vardepvalueexclude] .= "| buildhistory_emit_pkghistory" 55 56# Similarly for our function that gets the output signatures 57SSTATEPOSTUNPACKFUNCS:append = " buildhistory_emit_outputsigs" 58sstate_installpkgdir[vardepsexclude] += "buildhistory_emit_outputsigs" 59SSTATEPOSTUNPACKFUNCS[vardepvalueexclude] .= "| buildhistory_emit_outputsigs" 60 61# All items excepts those listed here will be removed from a recipe's 62# build history directory by buildhistory_emit_pkghistory(). This is 63# necessary because some of these items (package directories, files that 64# we no longer emit) might be obsolete. 65# 66# When extending build history, derive your class from buildhistory.bbclass 67# and extend this list here with the additional files created by the derived 68# class. 69BUILDHISTORY_PRESERVE = "latest latest_srcrev sysroot" 70 71PATCH_GIT_USER_EMAIL ?= "buildhistory@oe" 72PATCH_GIT_USER_NAME ?= "OpenEmbedded" 73 74# 75# Write out the contents of the sysroot 76# 77buildhistory_emit_sysroot() { 78 mkdir --parents ${BUILDHISTORY_DIR_PACKAGE} 79 case ${CLASSOVERRIDE} in 80 class-native|class-cross|class-crosssdk) 81 BASE=${SYSROOT_DESTDIR}/${STAGING_DIR_NATIVE} 82 ;; 83 *) 84 BASE=${SYSROOT_DESTDIR} 85 ;; 86 esac 87 buildhistory_list_files_no_owners $BASE ${BUILDHISTORY_DIR_PACKAGE}/sysroot 88} 89 90# 91# Write out metadata about this package for comparison when writing future packages 92# 93python buildhistory_emit_pkghistory() { 94 if d.getVar('BB_CURRENTTASK') in ['populate_sysroot', 'populate_sysroot_setscene']: 95 bb.build.exec_func("buildhistory_emit_sysroot", d) 96 return 0 97 98 if not "package" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): 99 return 0 100 101 if d.getVar('BB_CURRENTTASK') in ['package', 'package_setscene']: 102 # Create files-in-<package-name>.txt files containing a list of files of each recipe's package 103 bb.build.exec_func("buildhistory_list_pkg_files", d) 104 return 0 105 106 if not d.getVar('BB_CURRENTTASK') in ['packagedata', 'packagedata_setscene']: 107 return 0 108 109 import re 110 import json 111 import shlex 112 import errno 113 114 pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') 115 oldpkghistdir = d.getVar('BUILDHISTORY_OLD_DIR_PACKAGE') 116 117 class RecipeInfo: 118 def __init__(self, name): 119 self.name = name 120 self.pe = "0" 121 self.pv = "0" 122 self.pr = "r0" 123 self.depends = "" 124 self.packages = "" 125 self.srcrev = "" 126 self.layer = "" 127 self.license = "" 128 self.config = "" 129 self.src_uri = "" 130 131 132 class PackageInfo: 133 def __init__(self, name): 134 self.name = name 135 self.pe = "0" 136 self.pv = "0" 137 self.pr = "r0" 138 # pkg/pkge/pkgv/pkgr should be empty because we want to be able to default them 139 self.pkg = "" 140 self.pkge = "" 141 self.pkgv = "" 142 self.pkgr = "" 143 self.size = 0 144 self.depends = "" 145 self.rprovides = "" 146 self.rdepends = "" 147 self.rrecommends = "" 148 self.rsuggests = "" 149 self.rreplaces = "" 150 self.rconflicts = "" 151 self.files = "" 152 self.filelist = "" 153 # Variables that need to be written to their own separate file 154 self.filevars = dict.fromkeys(['pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm']) 155 156 # Should check PACKAGES here to see if anything removed 157 158 def readPackageInfo(pkg, histfile): 159 pkginfo = PackageInfo(pkg) 160 with open(histfile, "r") as f: 161 for line in f: 162 lns = line.split('=', 1) 163 name = lns[0].strip() 164 value = lns[1].strip(" \t\r\n").strip('"') 165 if name == "PE": 166 pkginfo.pe = value 167 elif name == "PV": 168 pkginfo.pv = value 169 elif name == "PR": 170 pkginfo.pr = value 171 elif name == "PKG": 172 pkginfo.pkg = value 173 elif name == "PKGE": 174 pkginfo.pkge = value 175 elif name == "PKGV": 176 pkginfo.pkgv = value 177 elif name == "PKGR": 178 pkginfo.pkgr = value 179 elif name == "RPROVIDES": 180 pkginfo.rprovides = value 181 elif name == "RDEPENDS": 182 pkginfo.rdepends = value 183 elif name == "RRECOMMENDS": 184 pkginfo.rrecommends = value 185 elif name == "RSUGGESTS": 186 pkginfo.rsuggests = value 187 elif name == "RREPLACES": 188 pkginfo.rreplaces = value 189 elif name == "RCONFLICTS": 190 pkginfo.rconflicts = value 191 elif name == "PKGSIZE": 192 pkginfo.size = int(value) 193 elif name == "FILES": 194 pkginfo.files = value 195 elif name == "FILELIST": 196 pkginfo.filelist = value 197 # Apply defaults 198 if not pkginfo.pkg: 199 pkginfo.pkg = pkginfo.name 200 if not pkginfo.pkge: 201 pkginfo.pkge = pkginfo.pe 202 if not pkginfo.pkgv: 203 pkginfo.pkgv = pkginfo.pv 204 if not pkginfo.pkgr: 205 pkginfo.pkgr = pkginfo.pr 206 return pkginfo 207 208 def getlastpkgversion(pkg): 209 try: 210 histfile = os.path.join(oldpkghistdir, pkg, "latest") 211 return readPackageInfo(pkg, histfile) 212 except EnvironmentError: 213 return None 214 215 def sortpkglist(string): 216 pkgiter = re.finditer(r'[a-zA-Z0-9.+-]+( \([><=]+[^)]+\))?', string, 0) 217 pkglist = [p.group(0) for p in pkgiter] 218 pkglist.sort() 219 return ' '.join(pkglist) 220 221 def sortlist(string): 222 items = string.split(' ') 223 items.sort() 224 return ' '.join(items) 225 226 pn = d.getVar('PN') 227 pe = d.getVar('PE') or "0" 228 pv = d.getVar('PV') 229 pr = d.getVar('PR') 230 layer = bb.utils.get_file_layer(d.getVar('FILE'), d) 231 license = d.getVar('LICENSE') 232 233 pkgdata_dir = d.getVar('PKGDATA_DIR') 234 packages = "" 235 try: 236 with open(os.path.join(pkgdata_dir, pn)) as f: 237 for line in f.readlines(): 238 if line.startswith('PACKAGES: '): 239 packages = oe.utils.squashspaces(line.split(': ', 1)[1]) 240 break 241 except IOError as e: 242 if e.errno == errno.ENOENT: 243 # Probably a -cross recipe, just ignore 244 return 0 245 else: 246 raise 247 248 packagelist = packages.split() 249 preserve = d.getVar('BUILDHISTORY_PRESERVE').split() 250 if not os.path.exists(pkghistdir): 251 bb.utils.mkdirhier(pkghistdir) 252 else: 253 # Remove files for packages that no longer exist 254 for item in os.listdir(pkghistdir): 255 if item not in preserve: 256 if item not in packagelist: 257 itempath = os.path.join(pkghistdir, item) 258 if os.path.isdir(itempath): 259 for subfile in os.listdir(itempath): 260 os.unlink(os.path.join(itempath, subfile)) 261 os.rmdir(itempath) 262 else: 263 os.unlink(itempath) 264 265 rcpinfo = RecipeInfo(pn) 266 rcpinfo.pe = pe 267 rcpinfo.pv = pv 268 rcpinfo.pr = pr 269 rcpinfo.depends = sortlist(oe.utils.squashspaces(d.getVar('DEPENDS') or "")) 270 rcpinfo.packages = packages 271 rcpinfo.layer = layer 272 rcpinfo.license = license 273 rcpinfo.config = sortlist(oe.utils.squashspaces(d.getVar('PACKAGECONFIG') or "")) 274 rcpinfo.src_uri = oe.utils.squashspaces(d.getVar('SRC_URI') or "") 275 write_recipehistory(rcpinfo, d) 276 277 bb.build.exec_func("read_subpackage_metadata", d) 278 279 for pkg in packagelist: 280 localdata = d.createCopy() 281 localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + pkg) 282 283 pkge = localdata.getVar("PKGE") or '0' 284 pkgv = localdata.getVar("PKGV") 285 pkgr = localdata.getVar("PKGR") 286 # 287 # Find out what the last version was 288 # Make sure the version did not decrease 289 # 290 lastversion = getlastpkgversion(pkg) 291 if lastversion: 292 last_pkge = lastversion.pkge 293 last_pkgv = lastversion.pkgv 294 last_pkgr = lastversion.pkgr 295 r = bb.utils.vercmp((pkge, pkgv, pkgr), (last_pkge, last_pkgv, last_pkgr)) 296 if r < 0: 297 msg = "Package version for package %s went backwards which would break package feeds (from %s:%s-%s to %s:%s-%s)" % (pkg, last_pkge, last_pkgv, last_pkgr, pkge, pkgv, pkgr) 298 oe.qa.handle_error("version-going-backwards", msg, d) 299 300 pkginfo = PackageInfo(pkg) 301 # Apparently the version can be different on a per-package basis (see Python) 302 pkginfo.pe = localdata.getVar("PE") or '0' 303 pkginfo.pv = localdata.getVar("PV") 304 pkginfo.pr = localdata.getVar("PR") 305 pkginfo.pkg = localdata.getVar("PKG") 306 pkginfo.pkge = pkge 307 pkginfo.pkgv = pkgv 308 pkginfo.pkgr = pkgr 309 pkginfo.rprovides = sortpkglist(oe.utils.squashspaces(localdata.getVar("RPROVIDES") or "")) 310 pkginfo.rdepends = sortpkglist(oe.utils.squashspaces(localdata.getVar("RDEPENDS") or "")) 311 pkginfo.rrecommends = sortpkglist(oe.utils.squashspaces(localdata.getVar("RRECOMMENDS") or "")) 312 pkginfo.rsuggests = sortpkglist(oe.utils.squashspaces(localdata.getVar("RSUGGESTS") or "")) 313 pkginfo.replaces = sortpkglist(oe.utils.squashspaces(localdata.getVar("RREPLACES") or "")) 314 pkginfo.rconflicts = sortpkglist(oe.utils.squashspaces(localdata.getVar("RCONFLICTS") or "")) 315 pkginfo.files = oe.utils.squashspaces(localdata.getVar("FILES") or "") 316 for filevar in pkginfo.filevars: 317 pkginfo.filevars[filevar] = localdata.getVar(filevar) or "" 318 319 # Gather information about packaged files 320 val = localdata.getVar('FILES_INFO') or '' 321 dictval = json.loads(val) 322 filelist = list(dictval.keys()) 323 filelist.sort() 324 pkginfo.filelist = " ".join([shlex.quote(x) for x in filelist]) 325 326 pkginfo.size = int(localdata.getVar('PKGSIZE') or '0') 327 328 write_pkghistory(pkginfo, d) 329 330 oe.qa.exit_if_errors(d) 331} 332 333python buildhistory_emit_outputsigs() { 334 if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): 335 return 336 337 import hashlib 338 339 taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task', 'output') 340 bb.utils.mkdirhier(taskoutdir) 341 currenttask = d.getVar('BB_CURRENTTASK') 342 pn = d.getVar('PN') 343 taskfile = os.path.join(taskoutdir, '%s.%s' % (pn, currenttask)) 344 345 cwd = os.getcwd() 346 filesigs = {} 347 for root, _, files in os.walk(cwd): 348 for fname in files: 349 if fname == 'fixmepath': 350 continue 351 fullpath = os.path.join(root, fname) 352 try: 353 if os.path.islink(fullpath): 354 sha256 = hashlib.sha256(os.readlink(fullpath).encode('utf-8')).hexdigest() 355 elif os.path.isfile(fullpath): 356 sha256 = bb.utils.sha256_file(fullpath) 357 else: 358 continue 359 except OSError: 360 bb.warn('buildhistory: unable to read %s to get output signature' % fullpath) 361 continue 362 filesigs[os.path.relpath(fullpath, cwd)] = sha256 363 with open(taskfile, 'w') as f: 364 for fpath, fsig in sorted(filesigs.items(), key=lambda item: item[0]): 365 f.write('%s %s\n' % (fpath, fsig)) 366} 367 368 369def write_recipehistory(rcpinfo, d): 370 bb.debug(2, "Writing recipe history") 371 372 pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') 373 374 infofile = os.path.join(pkghistdir, "latest") 375 with open(infofile, "w") as f: 376 if rcpinfo.pe != "0": 377 f.write(u"PE = %s\n" % rcpinfo.pe) 378 f.write(u"PV = %s\n" % rcpinfo.pv) 379 f.write(u"PR = %s\n" % rcpinfo.pr) 380 f.write(u"DEPENDS = %s\n" % rcpinfo.depends) 381 f.write(u"PACKAGES = %s\n" % rcpinfo.packages) 382 f.write(u"LAYER = %s\n" % rcpinfo.layer) 383 f.write(u"LICENSE = %s\n" % rcpinfo.license) 384 f.write(u"CONFIG = %s\n" % rcpinfo.config) 385 f.write(u"SRC_URI = %s\n" % rcpinfo.src_uri) 386 387 write_latest_srcrev(d, pkghistdir) 388 389def write_pkghistory(pkginfo, d): 390 bb.debug(2, "Writing package history for package %s" % pkginfo.name) 391 392 pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE') 393 394 pkgpath = os.path.join(pkghistdir, pkginfo.name) 395 if not os.path.exists(pkgpath): 396 bb.utils.mkdirhier(pkgpath) 397 398 infofile = os.path.join(pkgpath, "latest") 399 with open(infofile, "w") as f: 400 if pkginfo.pe != "0": 401 f.write(u"PE = %s\n" % pkginfo.pe) 402 f.write(u"PV = %s\n" % pkginfo.pv) 403 f.write(u"PR = %s\n" % pkginfo.pr) 404 405 if pkginfo.pkg != pkginfo.name: 406 f.write(u"PKG = %s\n" % pkginfo.pkg) 407 if pkginfo.pkge != pkginfo.pe: 408 f.write(u"PKGE = %s\n" % pkginfo.pkge) 409 if pkginfo.pkgv != pkginfo.pv: 410 f.write(u"PKGV = %s\n" % pkginfo.pkgv) 411 if pkginfo.pkgr != pkginfo.pr: 412 f.write(u"PKGR = %s\n" % pkginfo.pkgr) 413 f.write(u"RPROVIDES = %s\n" % pkginfo.rprovides) 414 f.write(u"RDEPENDS = %s\n" % pkginfo.rdepends) 415 f.write(u"RRECOMMENDS = %s\n" % pkginfo.rrecommends) 416 if pkginfo.rsuggests: 417 f.write(u"RSUGGESTS = %s\n" % pkginfo.rsuggests) 418 if pkginfo.rreplaces: 419 f.write(u"RREPLACES = %s\n" % pkginfo.rreplaces) 420 if pkginfo.rconflicts: 421 f.write(u"RCONFLICTS = %s\n" % pkginfo.rconflicts) 422 f.write(u"PKGSIZE = %d\n" % pkginfo.size) 423 f.write(u"FILES = %s\n" % pkginfo.files) 424 f.write(u"FILELIST = %s\n" % pkginfo.filelist) 425 426 for filevar in pkginfo.filevars: 427 filevarpath = os.path.join(pkgpath, "latest.%s" % filevar) 428 val = pkginfo.filevars[filevar] 429 if val: 430 with open(filevarpath, "w") as f: 431 f.write(val) 432 else: 433 if os.path.exists(filevarpath): 434 os.unlink(filevarpath) 435 436# 437# rootfs_type can be: image, sdk_target, sdk_host 438# 439def buildhistory_list_installed(d, rootfs_type="image"): 440 from oe.rootfs import image_list_installed_packages 441 from oe.sdk import sdk_list_installed_packages 442 from oe.utils import format_pkg_list 443 444 process_list = [('file', 'bh_installed_pkgs_%s.txt' % os.getpid()),\ 445 ('deps', 'bh_installed_pkgs_deps_%s.txt' % os.getpid())] 446 447 if rootfs_type == "image": 448 pkgs = image_list_installed_packages(d) 449 else: 450 pkgs = sdk_list_installed_packages(d, rootfs_type == "sdk_target") 451 452 if rootfs_type == "sdk_host": 453 pkgdata_dir = d.getVar('PKGDATA_DIR_SDK') 454 else: 455 pkgdata_dir = d.getVar('PKGDATA_DIR') 456 457 for output_type, output_file in process_list: 458 output_file_full = os.path.join(d.getVar('WORKDIR'), output_file) 459 460 with open(output_file_full, 'w') as output: 461 output.write(format_pkg_list(pkgs, output_type, pkgdata_dir)) 462 463python buildhistory_list_installed_image() { 464 buildhistory_list_installed(d) 465} 466 467python buildhistory_list_installed_sdk_target() { 468 buildhistory_list_installed(d, "sdk_target") 469} 470 471python buildhistory_list_installed_sdk_host() { 472 buildhistory_list_installed(d, "sdk_host") 473} 474 475buildhistory_get_installed() { 476 mkdir -p $1 477 478 # Get list of installed packages 479 pkgcache="$1/installed-packages.tmp" 480 cat ${WORKDIR}/bh_installed_pkgs_${PID}.txt | sort > $pkgcache && rm ${WORKDIR}/bh_installed_pkgs_${PID}.txt 481 482 cat $pkgcache | awk '{ print $1 }' > $1/installed-package-names.txt 483 484 if [ -s $pkgcache ] ; then 485 cat $pkgcache | awk '{ print $2 }' | xargs -n1 basename > $1/installed-packages.txt 486 else 487 printf "" > $1/installed-packages.txt 488 fi 489 490 # Produce dependency graph 491 # First, quote each name to handle characters that cause issues for dot 492 sed 's:\([^| ]*\):"\1":g' ${WORKDIR}/bh_installed_pkgs_deps_${PID}.txt > $1/depends.tmp && 493 rm ${WORKDIR}/bh_installed_pkgs_deps_${PID}.txt 494 # Remove lines with rpmlib(...) and config(...) dependencies, change the 495 # delimiter from pipe to "->", set the style for recommend lines and 496 # turn versioned dependencies into edge labels. 497 sed -i -e '/rpmlib(/d' \ 498 -e '/config(/d' \ 499 -e 's:|: -> :' \ 500 -e 's:"\[REC\]":[style=dotted]:' \ 501 -e 's:"\([<>=]\+\)" "\([^"]*\)":[label="\1 \2"]:' \ 502 -e 's:"\([*]\+\)" "\([^"]*\)":[label="\2"]:' \ 503 -e 's:"\[RPROVIDES\]":[style=dashed]:' \ 504 $1/depends.tmp 505 # Add header, sorted and de-duped contents and footer and then delete the temp file 506 printf "digraph depends {\n node [shape=plaintext]\n" > $1/depends.dot 507 cat $1/depends.tmp | sort -u >> $1/depends.dot 508 echo "}" >> $1/depends.dot 509 rm $1/depends.tmp 510 511 # Set correct pkgdatadir 512 pkgdatadir=${PKGDATA_DIR} 513 if [ "$2" = "sdk" ] && [ "$3" = "host" ] ; then 514 pkgdatadir="${PKGDATA_DIR_SDK}" 515 fi 516 517 # Produce installed package sizes list 518 oe-pkgdata-util -p $pkgdatadir read-value "PKGSIZE" -n -f $pkgcache > $1/installed-package-sizes.tmp 519 cat $1/installed-package-sizes.tmp | awk '{print $2 "\tKiB\t" $1}' | sort -n -r > $1/installed-package-sizes.txt 520 rm $1/installed-package-sizes.tmp 521 522 # Produce package info: runtime_name, buildtime_name, recipe, version, size 523 oe-pkgdata-util -p $pkgdatadir read-value "PACKAGE,PN,PV,PKGSIZE" -n -f $pkgcache > $1/installed-package-info.tmp 524 cat $1/installed-package-info.tmp | sort -n -r -k 5 > $1/installed-package-info.txt 525 rm $1/installed-package-info.tmp 526 527 # We're now done with the cache, delete it 528 rm $pkgcache 529 530 if [ "$2" != "sdk" ] ; then 531 # Produce some cut-down graphs (for readability) 532 grep -v kernel-image $1/depends.dot | grep -v kernel-3 | grep -v kernel-4 > $1/depends-nokernel.dot 533 grep -v libc6 $1/depends-nokernel.dot | grep -v libgcc > $1/depends-nokernel-nolibc.dot 534 grep -v update- $1/depends-nokernel-nolibc.dot > $1/depends-nokernel-nolibc-noupdate.dot 535 grep -v kernel-module $1/depends-nokernel-nolibc-noupdate.dot > $1/depends-nokernel-nolibc-noupdate-nomodules.dot 536 fi 537 538 # add complementary package information 539 if [ -e ${WORKDIR}/complementary_pkgs.txt ]; then 540 cp ${WORKDIR}/complementary_pkgs.txt $1 541 fi 542} 543 544buildhistory_get_image_installed() { 545 # Anything requiring the use of the packaging system should be done in here 546 # in case the packaging files are going to be removed for this image 547 548 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then 549 return 550 fi 551 552 buildhistory_get_installed ${BUILDHISTORY_DIR_IMAGE} 553} 554 555buildhistory_get_sdk_installed() { 556 # Anything requiring the use of the packaging system should be done in here 557 # in case the packaging files are going to be removed for this SDK 558 559 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then 560 return 561 fi 562 563 buildhistory_get_installed ${BUILDHISTORY_DIR_SDK}/$1 sdk $1 564} 565 566buildhistory_get_sdk_installed_host() { 567 buildhistory_get_sdk_installed host 568} 569 570buildhistory_get_sdk_installed_target() { 571 buildhistory_get_sdk_installed target 572} 573 574buildhistory_list_files() { 575 # List the files in the specified directory, but exclude date/time etc. 576 # This is somewhat messy, but handles where the size is not printed for device files under pseudo 577 ( cd $1 578 find_cmd='find . ! -path . -printf "%M %-10u %-10g %10s %p -> %l\n"' 579 if [ "$3" = "fakeroot" ] ; then 580 eval ${FAKEROOTENV} ${FAKEROOTCMD} $find_cmd 581 else 582 eval $find_cmd 583 fi | sort -k5 | sed 's/ * -> $//' > $2 ) 584} 585 586buildhistory_list_files_no_owners() { 587 # List the files in the specified directory, but exclude date/time etc. 588 # Also don't output the ownership data, but instead output just - - so 589 # that the same parsing code as for _list_files works. 590 # This is somewhat messy, but handles where the size is not printed for device files under pseudo 591 ( cd $1 592 find_cmd='find . ! -path . -printf "%M - - %10s %p -> %l\n"' 593 if [ "$3" = "fakeroot" ] ; then 594 eval ${FAKEROOTENV} ${FAKEROOTCMD} "$find_cmd" 595 else 596 eval "$find_cmd" 597 fi | sort -k5 | sed 's/ * -> $//' > $2 ) 598} 599 600buildhistory_list_pkg_files() { 601 # Create individual files-in-package for each recipe's package 602 for pkgdir in $(find ${PKGDEST}/* -maxdepth 0 -type d); do 603 pkgname=$(basename $pkgdir) 604 outfolder="${BUILDHISTORY_DIR_PACKAGE}/$pkgname" 605 outfile="$outfolder/files-in-package.txt" 606 # Make sure the output folder exists so we can create the file 607 if [ ! -d $outfolder ] ; then 608 bbdebug 2 "Folder $outfolder does not exist, file $outfile not created" 609 continue 610 fi 611 buildhistory_list_files $pkgdir $outfile fakeroot 612 done 613} 614 615buildhistory_get_imageinfo() { 616 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then 617 return 618 fi 619 620 mkdir -p ${BUILDHISTORY_DIR_IMAGE} 621 buildhistory_list_files ${IMAGE_ROOTFS} ${BUILDHISTORY_DIR_IMAGE}/files-in-image.txt 622 623 # Collect files requested in BUILDHISTORY_IMAGE_FILES 624 rm -rf ${BUILDHISTORY_DIR_IMAGE}/image-files 625 for f in ${BUILDHISTORY_IMAGE_FILES}; do 626 if [ -f ${IMAGE_ROOTFS}/$f ] ; then 627 mkdir -p ${BUILDHISTORY_DIR_IMAGE}/image-files/`dirname $f` 628 cp ${IMAGE_ROOTFS}/$f ${BUILDHISTORY_DIR_IMAGE}/image-files/$f 629 fi 630 done 631 632 # Record some machine-readable meta-information about the image 633 printf "" > ${BUILDHISTORY_DIR_IMAGE}/image-info.txt 634 cat >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt <<END 635${@buildhistory_get_imagevars(d)} 636END 637 imagesize=`du -ks ${IMAGE_ROOTFS} | awk '{ print $1 }'` 638 echo "IMAGESIZE = $imagesize" >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt 639 640 # Add some configuration information 641 echo "${MACHINE}: ${IMAGE_BASENAME} configured for ${DISTRO} ${DISTRO_VERSION}" > ${BUILDHISTORY_DIR_IMAGE}/build-id.txt 642 643 cat >> ${BUILDHISTORY_DIR_IMAGE}/build-id.txt <<END 644${@buildhistory_get_build_id(d)} 645END 646} 647 648buildhistory_get_sdkinfo() { 649 if [ "${@bb.utils.contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then 650 return 651 fi 652 653 buildhistory_list_files ${SDK_OUTPUT} ${BUILDHISTORY_DIR_SDK}/files-in-sdk.txt 654 655 # Collect files requested in BUILDHISTORY_SDK_FILES 656 rm -rf ${BUILDHISTORY_DIR_SDK}/sdk-files 657 for f in ${BUILDHISTORY_SDK_FILES}; do 658 if [ -f ${SDK_OUTPUT}/${SDKPATH}/$f ] ; then 659 mkdir -p ${BUILDHISTORY_DIR_SDK}/sdk-files/`dirname $f` 660 cp ${SDK_OUTPUT}/${SDKPATH}/$f ${BUILDHISTORY_DIR_SDK}/sdk-files/$f 661 fi 662 done 663 664 # Record some machine-readable meta-information about the SDK 665 printf "" > ${BUILDHISTORY_DIR_SDK}/sdk-info.txt 666 cat >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt <<END 667${@buildhistory_get_sdkvars(d)} 668END 669 sdksize=`du -ks ${SDK_OUTPUT} | awk '{ print $1 }'` 670 echo "SDKSIZE = $sdksize" >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt 671} 672 673python buildhistory_get_extra_sdkinfo() { 674 import operator 675 from oe.sdk import get_extra_sdkinfo 676 677 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache') 678 extra_info = get_extra_sdkinfo(sstate_dir) 679 680 if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext' and \ 681 "sdk" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): 682 with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-package-sizes.txt'), 'w') as f: 683 filesizes_sorted = sorted(extra_info['filesizes'].items(), key=operator.itemgetter(1, 0), reverse=True) 684 for fn, size in filesizes_sorted: 685 f.write('%10d KiB %s\n' % (size, fn)) 686 with open(d.expand('${BUILDHISTORY_DIR_SDK}/sstate-task-sizes.txt'), 'w') as f: 687 tasksizes_sorted = sorted(extra_info['tasksizes'].items(), key=operator.itemgetter(1, 0), reverse=True) 688 for task, size in tasksizes_sorted: 689 f.write('%10d KiB %s\n' % (size, task)) 690} 691 692# By using ROOTFS_POSTUNINSTALL_COMMAND we get in after uninstallation of 693# unneeded packages but before the removal of packaging files 694ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_list_installed_image" 695ROOTFS_POSTUNINSTALL_COMMAND += "buildhistory_get_image_installed" 696ROOTFS_POSTUNINSTALL_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_image| buildhistory_get_image_installed" 697ROOTFS_POSTUNINSTALL_COMMAND[vardepsexclude] += "buildhistory_list_installed_image buildhistory_get_image_installed" 698 699IMAGE_POSTPROCESS_COMMAND += "buildhistory_get_imageinfo" 700IMAGE_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_imageinfo" 701IMAGE_POSTPROCESS_COMMAND[vardepsexclude] += "buildhistory_get_imageinfo" 702 703# We want these to be the last run so that we get called after complementary package installation 704POPULATE_SDK_POST_TARGET_COMMAND:append = " buildhistory_list_installed_sdk_target" 705POPULATE_SDK_POST_TARGET_COMMAND:append = " buildhistory_get_sdk_installed_target" 706POPULATE_SDK_POST_TARGET_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_target| buildhistory_get_sdk_installed_target" 707POPULATE_SDK_POST_TARGET_COMMAND[vardepsexclude] += "buildhistory_list_installed_sdk_target buildhistory_get_sdk_installed_target" 708 709POPULATE_SDK_POST_HOST_COMMAND:append = " buildhistory_list_installed_sdk_host" 710POPULATE_SDK_POST_HOST_COMMAND:append = " buildhistory_get_sdk_installed_host" 711POPULATE_SDK_POST_HOST_COMMAND[vardepvalueexclude] .= "| buildhistory_list_installed_sdk_host| buildhistory_get_sdk_installed_host" 712POPULATE_SDK_POST_HOST_COMMAND[vardepsexclude] += "buildhistory_list_installed_sdk_host buildhistory_get_sdk_installed_host" 713 714SDK_POSTPROCESS_COMMAND:append = " buildhistory_get_sdkinfo buildhistory_get_extra_sdkinfo" 715SDK_POSTPROCESS_COMMAND[vardepvalueexclude] .= "| buildhistory_get_sdkinfo buildhistory_get_extra_sdkinfo" 716SDK_POSTPROCESS_COMMAND[vardepsexclude] += "buildhistory_get_sdkinfo buildhistory_get_extra_sdkinfo" 717 718python buildhistory_write_sigs() { 719 if not "task" in (d.getVar('BUILDHISTORY_FEATURES') or "").split(): 720 return 721 722 # Create sigs file 723 if hasattr(bb.parse.siggen, 'dump_siglist'): 724 taskoutdir = os.path.join(d.getVar('BUILDHISTORY_DIR'), 'task') 725 bb.utils.mkdirhier(taskoutdir) 726 bb.parse.siggen.dump_siglist(os.path.join(taskoutdir, 'tasksigs.txt'), d.getVar("BUILDHISTORY_PATH_PREFIX_STRIP")) 727} 728 729def buildhistory_get_build_id(d): 730 if d.getVar('BB_WORKERCONTEXT') != '1': 731 return "" 732 localdata = bb.data.createCopy(d) 733 statuslines = [] 734 for func in oe.data.typed_value('BUILDCFG_FUNCS', localdata): 735 g = globals() 736 if func not in g: 737 bb.warn("Build configuration function '%s' does not exist" % func) 738 else: 739 flines = g[func](localdata) 740 if flines: 741 statuslines.extend(flines) 742 743 statusheader = d.getVar('BUILDCFG_HEADER') 744 return('\n%s\n%s\n' % (statusheader, '\n'.join(statuslines))) 745 746def buildhistory_get_metadata_revs(d): 747 # We want an easily machine-readable format here 748 revisions = oe.buildcfg.get_layer_revisions(d) 749 medadata_revs = ["%-17s = %s:%s%s" % (r[1], r[2], r[3], r[4]) for r in revisions] 750 return '\n'.join(medadata_revs) 751 752def outputvars(vars, listvars, d): 753 vars = vars.split() 754 listvars = listvars.split() 755 ret = "" 756 for var in vars: 757 value = d.getVar(var) or "" 758 if var in listvars: 759 # Squash out spaces 760 value = oe.utils.squashspaces(value) 761 ret += "%s = %s\n" % (var, value) 762 return ret.rstrip('\n') 763 764def buildhistory_get_imagevars(d): 765 if d.getVar('BB_WORKERCONTEXT') != '1': 766 return "" 767 imagevars = "DISTRO DISTRO_VERSION USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE ROOTFS_POSTPROCESS_COMMAND IMAGE_POSTPROCESS_COMMAND" 768 listvars = "USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS PACKAGE_EXCLUDE" 769 return outputvars(imagevars, listvars, d) 770 771def buildhistory_get_sdkvars(d): 772 if d.getVar('BB_WORKERCONTEXT') != '1': 773 return "" 774 sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE SDKIMAGE_FEATURES TOOLCHAIN_HOST_TASK TOOLCHAIN_TARGET_TASK BAD_RECOMMENDATIONS NO_RECOMMENDATIONS PACKAGE_EXCLUDE" 775 if d.getVar('BB_CURRENTTASK') == 'populate_sdk_ext': 776 # Extensible SDK uses some additional variables 777 sdkvars += " ESDK_LOCALCONF_ALLOW ESDK_LOCALCONF_REMOVE ESDK_CLASS_INHERIT_DISABLE SDK_UPDATE_URL SDK_EXT_TYPE SDK_RECRDEP_TASKS SDK_INCLUDE_PKGDATA SDK_INCLUDE_TOOLCHAIN" 778 listvars = "SDKIMAGE_FEATURES BAD_RECOMMENDATIONS PACKAGE_EXCLUDE ESDK_LOCALCONF_ALLOW ESDK_LOCALCONF_REMOVE ESDK_CLASS_INHERIT_DISABLE" 779 return outputvars(sdkvars, listvars, d) 780 781 782def buildhistory_get_cmdline(d): 783 argv = d.getVar('BB_CMDLINE', False) 784 if argv: 785 if argv[0].endswith('bin/bitbake'): 786 bincmd = 'bitbake' 787 else: 788 bincmd = argv[0] 789 return '%s %s' % (bincmd, ' '.join(argv[1:])) 790 return '' 791 792 793buildhistory_single_commit() { 794 if [ "$3" = "" ] ; then 795 commitopts="${BUILDHISTORY_DIR}/ --allow-empty" 796 shortlogprefix="No changes: " 797 else 798 commitopts="" 799 shortlogprefix="" 800 fi 801 if [ "${BUILDHISTORY_BUILD_FAILURES}" = "0" ] ; then 802 result="succeeded" 803 else 804 result="failed" 805 fi 806 case ${BUILDHISTORY_BUILD_INTERRUPTED} in 807 1) 808 result="$result (interrupted)" 809 ;; 810 2) 811 result="$result (force interrupted)" 812 ;; 813 esac 814 commitmsgfile=`mktemp` 815 cat > $commitmsgfile << END 816${shortlogprefix}Build ${BUILDNAME} of ${DISTRO} ${DISTRO_VERSION} for machine ${MACHINE} on $2 817 818cmd: $1 819 820result: $result 821 822metadata revisions: 823END 824 cat ${BUILDHISTORY_DIR}/metadata-revs >> $commitmsgfile 825 git commit $commitopts -F $commitmsgfile --author "${BUILDHISTORY_COMMIT_AUTHOR}" > /dev/null 826 rm $commitmsgfile 827} 828 829buildhistory_commit() { 830 if [ ! -d ${BUILDHISTORY_DIR} ] ; then 831 # Code above that creates this dir never executed, so there can't be anything to commit 832 return 833 fi 834 835 # Create a machine-readable list of metadata revisions for each layer 836 cat > ${BUILDHISTORY_DIR}/metadata-revs <<END 837${@buildhistory_get_metadata_revs(d)} 838END 839 840 ( cd ${BUILDHISTORY_DIR}/ 841 # Initialise the repo if necessary 842 if [ ! -e .git ] ; then 843 git init -q 844 else 845 git tag -f ${BUILDHISTORY_TAG}-minus-3 ${BUILDHISTORY_TAG}-minus-2 > /dev/null 2>&1 || true 846 git tag -f ${BUILDHISTORY_TAG}-minus-2 ${BUILDHISTORY_TAG}-minus-1 > /dev/null 2>&1 || true 847 git tag -f ${BUILDHISTORY_TAG}-minus-1 > /dev/null 2>&1 || true 848 fi 849 850 check_git_config 851 852 # Check if there are new/changed files to commit (other than metadata-revs) 853 repostatus=`git status --porcelain | grep -v " metadata-revs$"` 854 HOSTNAME=`hostname 2>/dev/null || echo unknown` 855 CMDLINE="${@buildhistory_get_cmdline(d)}" 856 if [ "$repostatus" != "" ] ; then 857 git add -A . 858 # porcelain output looks like "?? packages/foo/bar" 859 # Ensure we commit metadata-revs with the first commit 860 buildhistory_single_commit "$CMDLINE" "$HOSTNAME" dummy 861 git gc --auto --quiet 862 else 863 buildhistory_single_commit "$CMDLINE" "$HOSTNAME" 864 fi 865 if [ "${BUILDHISTORY_PUSH_REPO}" != "" ] ; then 866 git push -q ${BUILDHISTORY_PUSH_REPO} 867 fi) || true 868} 869 870python buildhistory_eventhandler() { 871 if (e.data.getVar('BUILDHISTORY_FEATURES') or "").strip(): 872 reset = e.data.getVar("BUILDHISTORY_RESET") 873 olddir = e.data.getVar("BUILDHISTORY_OLD_DIR") 874 if isinstance(e, bb.event.BuildStarted): 875 if reset: 876 import shutil 877 # Clean up after potentially interrupted build. 878 if os.path.isdir(olddir): 879 shutil.rmtree(olddir) 880 rootdir = e.data.getVar("BUILDHISTORY_DIR") 881 bb.utils.mkdirhier(rootdir) 882 entries = [ x for x in os.listdir(rootdir) if not x.startswith('.') ] 883 bb.utils.mkdirhier(olddir) 884 for entry in entries: 885 bb.utils.rename(os.path.join(rootdir, entry), 886 os.path.join(olddir, entry)) 887 elif isinstance(e, bb.event.BuildCompleted): 888 if reset: 889 import shutil 890 shutil.rmtree(olddir) 891 if e.data.getVar("BUILDHISTORY_COMMIT") == "1": 892 bb.note("Writing buildhistory") 893 bb.build.exec_func("buildhistory_write_sigs", d) 894 import time 895 start=time.time() 896 localdata = bb.data.createCopy(e.data) 897 localdata.setVar('BUILDHISTORY_BUILD_FAILURES', str(e._failures)) 898 interrupted = getattr(e, '_interrupted', 0) 899 localdata.setVar('BUILDHISTORY_BUILD_INTERRUPTED', str(interrupted)) 900 bb.build.exec_func("buildhistory_commit", localdata) 901 stop=time.time() 902 bb.note("Writing buildhistory took: %s seconds" % round(stop-start)) 903 else: 904 bb.note("No commit since BUILDHISTORY_COMMIT != '1'") 905} 906 907addhandler buildhistory_eventhandler 908buildhistory_eventhandler[eventmask] = "bb.event.BuildCompleted bb.event.BuildStarted" 909 910 911# FIXME this ought to be moved into the fetcher 912def _get_srcrev_values(d): 913 """ 914 Return the version strings for the current recipe 915 """ 916 917 scms = [] 918 fetcher = bb.fetch.Fetch(d.getVar('SRC_URI').split(), d) 919 urldata = fetcher.ud 920 for u in urldata: 921 if urldata[u].method.supports_srcrev(): 922 scms.append(u) 923 924 dict_srcrevs = {} 925 dict_tag_srcrevs = {} 926 for scm in scms: 927 ud = urldata[scm] 928 for name in ud.names: 929 autoinc, rev = ud.method.sortable_revision(ud, d, name) 930 dict_srcrevs[name] = rev 931 if 'tag' in ud.parm: 932 tag = ud.parm['tag']; 933 key = name+'_'+tag 934 dict_tag_srcrevs[key] = rev 935 return (dict_srcrevs, dict_tag_srcrevs) 936 937do_fetch[postfuncs] += "write_srcrev" 938do_fetch[vardepsexclude] += "write_srcrev" 939python write_srcrev() { 940 write_latest_srcrev(d, d.getVar('BUILDHISTORY_DIR_PACKAGE')) 941} 942 943def write_latest_srcrev(d, pkghistdir): 944 srcrevfile = os.path.join(pkghistdir, 'latest_srcrev') 945 946 srcrevs, tag_srcrevs = _get_srcrev_values(d) 947 if srcrevs: 948 if not os.path.exists(pkghistdir): 949 bb.utils.mkdirhier(pkghistdir) 950 old_tag_srcrevs = {} 951 if os.path.exists(srcrevfile): 952 with open(srcrevfile) as f: 953 for line in f: 954 if line.startswith('# tag_'): 955 key, value = line.split("=", 1) 956 key = key.replace('# tag_', '').strip() 957 value = value.replace('"', '').strip() 958 old_tag_srcrevs[key] = value 959 with open(srcrevfile, 'w') as f: 960 for name, srcrev in sorted(srcrevs.items()): 961 suffix = "_" + name 962 if name == "default": 963 suffix = "" 964 orig_srcrev = d.getVar('SRCREV%s' % suffix, False) 965 if orig_srcrev: 966 f.write('# SRCREV%s = "%s"\n' % (suffix, orig_srcrev)) 967 f.write('SRCREV%s = "%s"\n' % (suffix, srcrev)) 968 for name, srcrev in sorted(tag_srcrevs.items()): 969 f.write('# tag_%s = "%s"\n' % (name, srcrev)) 970 if name in old_tag_srcrevs and old_tag_srcrevs[name] != srcrev: 971 pkg = d.getVar('PN') 972 bb.warn("Revision for tag %s in package %s was changed since last build (from %s to %s)" % (name, pkg, old_tag_srcrevs[name], srcrev)) 973 974 else: 975 if os.path.exists(srcrevfile): 976 os.remove(srcrevfile) 977 978do_testimage[postfuncs] += "write_ptest_result" 979do_testimage[vardepsexclude] += "write_ptest_result" 980 981python write_ptest_result() { 982 write_latest_ptest_result(d, d.getVar('BUILDHISTORY_DIR')) 983} 984 985def write_latest_ptest_result(d, histdir): 986 import glob 987 import subprocess 988 test_log_dir = d.getVar('TEST_LOG_DIR') 989 input_ptest = os.path.join(test_log_dir, 'ptest_log') 990 output_ptest = os.path.join(histdir, 'ptest') 991 if os.path.exists(input_ptest): 992 try: 993 # Lock it avoid race issue 994 lock = bb.utils.lockfile(output_ptest + "/ptest.lock") 995 bb.utils.mkdirhier(output_ptest) 996 oe.path.copytree(input_ptest, output_ptest) 997 # Sort test result 998 for result in glob.glob('%s/pass.fail.*' % output_ptest): 999 bb.debug(1, 'Processing %s' % result) 1000 cmd = ['sort', result, '-o', result] 1001 bb.debug(1, 'Running %s' % cmd) 1002 ret = subprocess.call(cmd) 1003 if ret != 0: 1004 bb.error('Failed to run %s!' % cmd) 1005 finally: 1006 bb.utils.unlockfile(lock) 1007