1#
2# Toaster helper class
3#
4# Copyright (C) 2013 Intel Corporation
5#
6# SPDX-License-Identifier: MIT
7#
8# This bbclass is designed to extract data used by OE-Core during the build process,
9# for recording in the Toaster system.
10# The data access is synchronous, preserving the build data integrity across
11# different builds.
12#
13# The data is transferred through the event system, using the MetadataEvent objects.
14#
15# The model is to enable the datadump functions as postfuncs, and have the dump
16# executed after the real taskfunc has been executed. This prevents task signature changing
17# is toaster is enabled or not. Build performance is not affected if Toaster is not enabled.
18#
19# To enable, use INHERIT in local.conf:
20#
21#       INHERIT += "toaster"
22#
23#
24#
25#
26
27# Find and dump layer info when we got the layers parsed
28
29
30
31python toaster_layerinfo_dumpdata() {
32    import subprocess
33
34    def _get_git_branch(layer_path):
35        branch = subprocess.Popen("git symbolic-ref HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0]
36        branch = branch.decode('utf-8')
37        branch = branch.replace('refs/heads/', '').rstrip()
38        return branch
39
40    def _get_git_revision(layer_path):
41        revision = subprocess.Popen("git rev-parse HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0].rstrip()
42        return revision
43
44    def _get_url_map_name(layer_name):
45        """ Some layers have a different name on openembedded.org site,
46            this method returns the correct name to use in the URL
47        """
48
49        url_name = layer_name
50        url_mapping = {'meta': 'openembedded-core'}
51
52        for key in url_mapping.keys():
53            if key == layer_name:
54                url_name = url_mapping[key]
55
56        return url_name
57
58    def _get_layer_version_information(layer_path):
59
60        layer_version_info = {}
61        layer_version_info['branch'] = _get_git_branch(layer_path)
62        layer_version_info['commit'] = _get_git_revision(layer_path)
63        layer_version_info['priority'] = 0
64
65        return layer_version_info
66
67
68    def _get_layer_dict(layer_path):
69
70        layer_info = {}
71        layer_name = layer_path.split('/')[-1]
72        layer_url = 'http://layers.openembedded.org/layerindex/layer/{layer}/'
73        layer_url_name = _get_url_map_name(layer_name)
74
75        layer_info['name'] = layer_url_name
76        layer_info['local_path'] = layer_path
77        layer_info['layer_index_url'] = layer_url.format(layer=layer_url_name)
78        layer_info['version'] = _get_layer_version_information(layer_path)
79
80        return layer_info
81
82
83    bblayers = e.data.getVar("BBLAYERS")
84
85    llayerinfo = {}
86
87    for layer in { l for l in bblayers.strip().split(" ") if len(l) }:
88        llayerinfo[layer] = _get_layer_dict(layer)
89
90
91    bb.event.fire(bb.event.MetadataEvent("LayerInfo", llayerinfo), e.data)
92}
93
94# Dump package file info data
95
96def _toaster_load_pkgdatafile(dirpath, filepath):
97    import json
98    import re
99    pkgdata = {}
100    with open(os.path.join(dirpath, filepath), "r") as fin:
101        for line in fin:
102            try:
103                kn, kv = line.strip().split(": ", 1)
104                m = re.match(r"^PKG:([^A-Z:]*)", kn)
105                if m:
106                    pkgdata['OPKGN'] = m.group(1)
107                kn = kn.split(":")[0]
108                pkgdata[kn] = kv
109                if kn.startswith('FILES_INFO'):
110                    pkgdata[kn] = json.loads(kv)
111
112            except ValueError:
113                pass    # ignore lines without valid key: value pairs
114    return pkgdata
115
116def _toaster_dumpdata(pkgdatadir, d):
117    """
118    Dumps the data about the packages created by a recipe
119    """
120
121    # No need to try and dumpdata if the recipe isn't generating packages
122    if not d.getVar('PACKAGES'):
123        return
124
125    lpkgdata = {}
126    datadir = os.path.join(pkgdatadir, 'runtime')
127
128    # scan and send data for each generated package
129    if os.path.exists(datadir):
130        for datafile in os.listdir(datadir):
131            if not datafile.endswith('.packaged'):
132                lpkgdata = _toaster_load_pkgdatafile(datadir, datafile)
133                # Fire an event containing the pkg data
134                bb.event.fire(bb.event.MetadataEvent("SinglePackageInfo", lpkgdata), d)
135
136python toaster_package_dumpdata() {
137    _toaster_dumpdata(d.getVar('PKGDESTWORK'), d)
138}
139
140python toaster_packagedata_dumpdata() {
141    # This path needs to match do_packagedata[sstate-inputdirs]
142    _toaster_dumpdata(os.path.join(d.getVar('WORKDIR'), 'pkgdata-pdata-input'), d)
143}
144
145# 2. Dump output image files information
146
147python toaster_artifact_dumpdata() {
148    """
149    Dump data about SDK variables
150    """
151
152    event_data = {
153      "TOOLCHAIN_OUTPUTNAME": d.getVar("TOOLCHAIN_OUTPUTNAME")
154    }
155
156    bb.event.fire(bb.event.MetadataEvent("SDKArtifactInfo", event_data), d)
157}
158
159# collect list of buildstats files based on fired events; when the build completes, collect all stats and fire an event with collected data
160
161python toaster_collect_task_stats() {
162    import bb.build
163    import bb.event
164    import bb.data
165    import bb.utils
166    import os
167
168    if not e.data.getVar('BUILDSTATS_BASE'):
169        return  # if we don't have buildstats, we cannot collect stats
170
171    toaster_statlist_file = os.path.join(e.data.getVar('BUILDSTATS_BASE'), "toasterstatlist")
172
173    def stat_to_float(value):
174        return float(value.strip('% \n\r'))
175
176    def _append_read_list(v):
177        lock = bb.utils.lockfile(e.data.expand("${TOPDIR}/toaster.lock"), False, True)
178
179        with open(toaster_statlist_file, "a") as fout:
180            taskdir = e.data.expand("${BUILDSTATS_BASE}/${BUILDNAME}/${PF}")
181            fout.write("%s::%s::%s::%s\n" % (e.taskfile, e.taskname, os.path.join(taskdir, e.task), e.data.expand("${PN}")))
182
183        bb.utils.unlockfile(lock)
184
185    def _read_stats(filename):
186        # seconds
187        cpu_time_user = 0
188        cpu_time_system = 0
189
190        # bytes
191        disk_io_read = 0
192        disk_io_write = 0
193
194        started = 0
195        ended = 0
196
197        taskname = ''
198
199        statinfo = {}
200
201        with open(filename, 'r') as task_bs:
202            for line in task_bs.readlines():
203                k,v = line.strip().split(": ", 1)
204                statinfo[k] = v
205
206        if "Started" in statinfo:
207            started = stat_to_float(statinfo["Started"])
208
209        if "Ended" in statinfo:
210            ended = stat_to_float(statinfo["Ended"])
211
212        if "Child rusage ru_utime" in statinfo:
213            cpu_time_user = cpu_time_user + stat_to_float(statinfo["Child rusage ru_utime"])
214
215        if "Child rusage ru_stime" in statinfo:
216            cpu_time_system = cpu_time_system + stat_to_float(statinfo["Child rusage ru_stime"])
217
218        if "IO write_bytes" in statinfo:
219            write_bytes = int(statinfo["IO write_bytes"].strip('% \n\r'))
220            disk_io_write = disk_io_write + write_bytes
221
222        if "IO read_bytes" in statinfo:
223            read_bytes = int(statinfo["IO read_bytes"].strip('% \n\r'))
224            disk_io_read = disk_io_read + read_bytes
225
226        return {
227            'stat_file': filename,
228            'cpu_time_user': cpu_time_user,
229            'cpu_time_system': cpu_time_system,
230            'disk_io_read': disk_io_read,
231            'disk_io_write': disk_io_write,
232            'started': started,
233            'ended': ended
234        }
235
236    if isinstance(e, (bb.build.TaskSucceeded, bb.build.TaskFailed)):
237        _append_read_list(e)
238        pass
239
240    if isinstance(e, bb.event.BuildCompleted) and os.path.exists(toaster_statlist_file):
241        events = []
242        with open(toaster_statlist_file, "r") as fin:
243            for line in fin:
244                (taskfile, taskname, filename, recipename) = line.strip().split("::")
245                stats = _read_stats(filename)
246                events.append((taskfile, taskname, stats, recipename))
247        bb.event.fire(bb.event.MetadataEvent("BuildStatsList", events), e.data)
248        os.unlink(toaster_statlist_file)
249}
250
251# dump relevant build history data as an event when the build is completed
252
253python toaster_buildhistory_dump() {
254    import re
255    BUILDHISTORY_DIR = e.data.expand("${TOPDIR}/buildhistory")
256    BUILDHISTORY_DIR_IMAGE_BASE = e.data.expand("%s/images/${MACHINE_ARCH}/${TCLIBC}/"% BUILDHISTORY_DIR)
257    pkgdata_dir = e.data.getVar("PKGDATA_DIR")
258
259
260    # scan the build targets for this build
261    images = {}
262    allpkgs = {}
263    files = {}
264    for target in e._pkgs:
265        target = target.split(':')[0] # strip ':<task>' suffix from the target
266        installed_img_path = e.data.expand(os.path.join(BUILDHISTORY_DIR_IMAGE_BASE, target))
267        if os.path.exists(installed_img_path):
268            images[target] = {}
269            files[target] = {}
270            files[target]['dirs'] = []
271            files[target]['syms'] = []
272            files[target]['files'] = []
273            with open("%s/installed-package-sizes.txt" % installed_img_path, "r") as fin:
274                for line in fin:
275                    line = line.rstrip(";")
276                    psize, punit, pname = line.split()
277                    # this size is "installed-size" as it measures how much space it takes on disk
278                    images[target][pname.strip()] = {'size':int(psize)*1024, 'depends' : []}
279
280            with open("%s/depends.dot" % installed_img_path, "r") as fin:
281                p = re.compile(r'\s*"(?P<name>[^"]+)"\s*->\s*"(?P<dep>[^"]+)"(?P<rec>.*?\[style=dotted\])?')
282                for line in fin:
283                    m = p.match(line)
284                    if not m:
285                        continue
286                    pname = m.group('name')
287                    dependsname = m.group('dep')
288                    deptype = 'recommends' if m.group('rec') else 'depends'
289
290                    # If RPM is used for packaging, then there may be
291                    # dependencies such as "/bin/sh", which will confuse
292                    # _toaster_load_pkgdatafile() later on. While at it, ignore
293                    # any dependencies that contain parentheses, e.g.,
294                    # "libc.so.6(GLIBC_2.7)".
295                    if dependsname.startswith('/') or '(' in dependsname:
296                        continue
297
298                    if not pname in images[target]:
299                        images[target][pname] = {'size': 0, 'depends' : []}
300                    if not dependsname in images[target]:
301                        images[target][dependsname] = {'size': 0, 'depends' : []}
302                    images[target][pname]['depends'].append((dependsname, deptype))
303
304            # files-in-image.txt is only generated if an image file is created,
305            # so the file entries ('syms', 'dirs', 'files') for a target will be
306            # empty for rootfs builds and other "image" tasks which don't
307            # produce image files
308            # (e.g. "bitbake core-image-minimal -c populate_sdk")
309            files_in_image_path = "%s/files-in-image.txt" % installed_img_path
310            if os.path.exists(files_in_image_path):
311                with open(files_in_image_path, "r") as fin:
312                    for line in fin:
313                        lc = [ x for x in line.strip().split(" ") if len(x) > 0 ]
314                        if lc[0].startswith("l"):
315                            files[target]['syms'].append(lc)
316                        elif lc[0].startswith("d"):
317                            files[target]['dirs'].append(lc)
318                        else:
319                            files[target]['files'].append(lc)
320
321            for pname in images[target]:
322                if not pname in allpkgs:
323                    try:
324                        pkgdata = _toaster_load_pkgdatafile("%s/runtime-reverse/" % pkgdata_dir, pname)
325                    except IOError as err:
326                        if err.errno == 2:
327                            # We expect this e.g. for RRECOMMENDS that are unsatisfied at runtime
328                            continue
329                        else:
330                            raise
331                    allpkgs[pname] = pkgdata
332
333
334    data = { 'pkgdata' : allpkgs, 'imgdata' : images, 'filedata' : files }
335
336    bb.event.fire(bb.event.MetadataEvent("ImagePkgList", data), e.data)
337
338}
339
340# get list of artifacts from sstate manifest
341python toaster_artifacts() {
342    if e.taskname in ["do_deploy", "do_image_complete", "do_populate_sdk", "do_populate_sdk_ext"]:
343        d2 = d.createCopy()
344        d2.setVar('FILE', e.taskfile)
345        # Use 'stamp-extra-info' if present, else use workaround
346        # to determine 'SSTATE_MANMACH'
347        extrainf = d2.getVarFlag(e.taskname, 'stamp-extra-info')
348        if extrainf:
349            d2.setVar('SSTATE_MANMACH', extrainf)
350        else:
351            if "do_populate_sdk" == e.taskname:
352                d2.setVar('SSTATE_MANMACH', d2.expand("${MACHINE}${SDKMACHINE}"))
353            else:
354                d2.setVar('SSTATE_MANMACH', d2.expand("${MACHINE}"))
355        manifest = oe.sstatesig.sstate_get_manifest_filename(e.taskname[3:], d2)[0]
356
357        if os.access(manifest, os.R_OK):
358            with open(manifest) as fmanifest:
359                artifacts = [fname.strip() for fname in fmanifest]
360                data = {"task": e.taskid, "artifacts": artifacts}
361                bb.event.fire(bb.event.MetadataEvent("TaskArtifacts", data), d2)
362}
363
364# set event handlers
365addhandler toaster_layerinfo_dumpdata
366toaster_layerinfo_dumpdata[eventmask] = "bb.event.TreeDataPreparationCompleted"
367
368addhandler toaster_collect_task_stats
369toaster_collect_task_stats[eventmask] = "bb.event.BuildCompleted bb.build.TaskSucceeded bb.build.TaskFailed"
370
371addhandler toaster_buildhistory_dump
372toaster_buildhistory_dump[eventmask] = "bb.event.BuildCompleted"
373
374addhandler toaster_artifacts
375toaster_artifacts[eventmask] = "bb.runqueue.runQueueTaskSkipped bb.runqueue.runQueueTaskCompleted"
376
377do_packagedata_setscene[postfuncs] += "toaster_packagedata_dumpdata "
378do_packagedata_setscene[vardepsexclude] += "toaster_packagedata_dumpdata "
379
380do_package[postfuncs] += "toaster_package_dumpdata "
381do_package[vardepsexclude] += "toaster_package_dumpdata "
382
383#do_populate_sdk[postfuncs] += "toaster_artifact_dumpdata "
384#do_populate_sdk[vardepsexclude] += "toaster_artifact_dumpdata "
385
386#do_populate_sdk_ext[postfuncs] += "toaster_artifact_dumpdata "
387#do_populate_sdk_ext[vardepsexclude] += "toaster_artifact_dumpdata "
388
389