1#
2# SPDX-License-Identifier: GPL-2.0-only
3#
4# This class should provide easy access to the different aspects of the
5# buildsystem such as layers, bitbake location, etc.
6#
7# SDK_LAYERS_EXCLUDE: Layers which will be excluded from SDK layers.
8# SDK_LAYERS_EXCLUDE_PATTERN: The simiar to SDK_LAYERS_EXCLUDE, this supports
9#                             python regular expression, use space as separator,
10#                              e.g.: ".*-downloads closed-.*"
11#
12
13import stat
14import shutil
15
16def _smart_copy(src, dest):
17    import subprocess
18    # smart_copy will choose the correct function depending on whether the
19    # source is a file or a directory.
20    mode = os.stat(src).st_mode
21    if stat.S_ISDIR(mode):
22        bb.utils.mkdirhier(dest)
23        cmd = "tar --exclude='.git' --exclude='__pycache__' --xattrs --xattrs-include='*' -chf - -C %s -p . \
24        | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest)
25        subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
26    else:
27        shutil.copyfile(src, dest)
28        shutil.copymode(src, dest)
29
30class BuildSystem(object):
31    def __init__(self, context, d):
32        self.d = d
33        self.context = context
34        self.layerdirs = [os.path.abspath(pth) for pth in d.getVar('BBLAYERS').split()]
35        self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE') or "").split()
36        self.layers_exclude_pattern = d.getVar('SDK_LAYERS_EXCLUDE_PATTERN')
37
38    def copy_bitbake_and_layers(self, destdir, workspace_name=None):
39        import re
40        # Copy in all metadata layers + bitbake (as repositories)
41        copied_corebase = None
42        layers_copied = []
43        bb.utils.mkdirhier(destdir)
44        layers = list(self.layerdirs)
45
46        corebase = os.path.abspath(self.d.getVar('COREBASE'))
47        layers.append(corebase)
48        # The bitbake build system uses the meta-skeleton layer as a layout
49        # for common recipies, e.g: the recipetool script to create kernel recipies
50        # Add the meta-skeleton layer to be included as part of the eSDK installation
51        layers.append(os.path.join(corebase, 'meta-skeleton'))
52
53        # Exclude layers
54        for layer_exclude in self.layers_exclude:
55            if layer_exclude in layers:
56                bb.note('Excluded %s from sdk layers since it is in SDK_LAYERS_EXCLUDE' % layer_exclude)
57                layers.remove(layer_exclude)
58
59        if self.layers_exclude_pattern:
60            layers_cp = layers[:]
61            for pattern in self.layers_exclude_pattern.split():
62                for layer in layers_cp:
63                    if re.match(pattern, layer):
64                        bb.note('Excluded %s from sdk layers since matched SDK_LAYERS_EXCLUDE_PATTERN' % layer)
65                        layers.remove(layer)
66
67        workspace_newname = workspace_name
68        if workspace_newname:
69            layernames = [os.path.basename(layer) for layer in layers]
70            extranum = 0
71            while workspace_newname in layernames:
72                extranum += 1
73                workspace_newname = '%s-%d' % (workspace_name, extranum)
74
75        corebase_files = self.d.getVar('COREBASE_FILES').split()
76        corebase_files = [corebase + '/' +x for x in corebase_files]
77        # Make sure bitbake goes in
78        bitbake_dir = bb.__file__.rsplit('/', 3)[0]
79        corebase_files.append(bitbake_dir)
80
81        for layer in layers:
82            layerconf = os.path.join(layer, 'conf', 'layer.conf')
83            layernewname = os.path.basename(layer)
84            workspace = False
85            if os.path.exists(layerconf):
86                with open(layerconf, 'r') as f:
87                    if f.readline().startswith("# ### workspace layer auto-generated by devtool ###"):
88                        if workspace_newname:
89                            layernewname = workspace_newname
90                            workspace = True
91                        else:
92                            bb.plain("NOTE: Excluding local workspace layer %s from %s" % (layer, self.context))
93                            continue
94
95            # If the layer was already under corebase, leave it there
96            # since layers such as meta have issues when moved.
97            layerdestpath = destdir
98            if corebase == os.path.dirname(layer):
99                layerdestpath += '/' + os.path.basename(corebase)
100            # If the layer is located somewhere under the same parent directory
101            # as corebase we keep the layer structure.
102            elif os.path.commonpath([layer, corebase]) == os.path.dirname(corebase):
103                layer_relative = os.path.relpath(layer, os.path.dirname(corebase))
104                if os.path.dirname(layer_relative) != layernewname:
105                    layerdestpath += '/' + os.path.dirname(layer_relative)
106
107            layerdestpath += '/' + layernewname
108
109            layer_relative = os.path.relpath(layerdestpath,
110                                             destdir)
111            # Treat corebase as special since it typically will contain
112            # build directories or other custom items.
113            if corebase == layer:
114                copied_corebase = layer_relative
115                bb.utils.mkdirhier(layerdestpath)
116                for f in corebase_files:
117                    f_basename = os.path.basename(f)
118                    destname = os.path.join(layerdestpath, f_basename)
119                    _smart_copy(f, destname)
120            else:
121                layers_copied.append(layer_relative)
122
123                if os.path.exists(os.path.join(layerdestpath, 'conf/layer.conf')):
124                    bb.note("Skipping layer %s, already handled" % layer)
125                else:
126                    _smart_copy(layer, layerdestpath)
127
128            if workspace:
129                # Make some adjustments original workspace layer
130                # Drop sources (recipe tasks will be locked, so we don't need them)
131                srcdir = os.path.join(layerdestpath, 'sources')
132                if os.path.isdir(srcdir):
133                    shutil.rmtree(srcdir)
134                # Drop all bbappends except the one for the image the SDK is being built for
135                # (because of externalsrc, the workspace bbappends will interfere with the
136                # locked signatures if present, and we don't need them anyway)
137                image_bbappend = os.path.splitext(os.path.basename(self.d.getVar('FILE')))[0] + '.bbappend'
138                appenddir = os.path.join(layerdestpath, 'appends')
139                if os.path.isdir(appenddir):
140                    for fn in os.listdir(appenddir):
141                        if fn == image_bbappend:
142                            continue
143                        else:
144                            os.remove(os.path.join(appenddir, fn))
145                # Drop README
146                readme = os.path.join(layerdestpath, 'README')
147                if os.path.exists(readme):
148                    os.remove(readme)
149                # Filter out comments in layer.conf and change layer name
150                layerconf = os.path.join(layerdestpath, 'conf', 'layer.conf')
151                with open(layerconf, 'r') as f:
152                    origlines = f.readlines()
153                with open(layerconf, 'w') as f:
154                    for line in origlines:
155                        if line.startswith('#'):
156                            continue
157                        line = line.replace('workspacelayer', workspace_newname)
158                        f.write(line)
159
160        # meta-skeleton layer is added as part of the build system
161        # but not as a layer included in the build, therefore it is
162        # not reported to the function caller.
163        for layer in layers_copied:
164            if layer.endswith('/meta-skeleton'):
165                layers_copied.remove(layer)
166                break
167
168        return copied_corebase, layers_copied
169
170def generate_locked_sigs(sigfile, d):
171    bb.utils.mkdirhier(os.path.dirname(sigfile))
172    depd = d.getVar('BB_TASKDEPDATA', False)
173    tasks = ['%s:%s' % (v[2], v[1]) for v in depd.values()]
174    bb.parse.siggen.dump_lockedsigs(sigfile, tasks)
175
176def prune_lockedsigs(excluded_tasks, excluded_targets, lockedsigs, onlynative, pruned_output):
177    with open(lockedsigs, 'r') as infile:
178        bb.utils.mkdirhier(os.path.dirname(pruned_output))
179        with open(pruned_output, 'w') as f:
180            invalue = False
181            for line in infile:
182                if invalue:
183                    if line.endswith('\\\n'):
184                        splitval = line.strip().split(':')
185                        if not splitval[1] in excluded_tasks and not splitval[0] in excluded_targets:
186                            if onlynative:
187                                if 'nativesdk' in splitval[0]:
188                                    f.write(line)
189                            else:
190                                f.write(line)
191                    else:
192                        f.write(line)
193                        invalue = False
194                elif line.startswith('SIGGEN_LOCKEDSIGS'):
195                    invalue = True
196                    f.write(line)
197
198def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_output, copy_output=None):
199    merged = {}
200    arch_order = []
201    with open(lockedsigs_main, 'r') as f:
202        invalue = None
203        for line in f:
204            if invalue:
205                if line.endswith('\\\n'):
206                    merged[invalue].append(line)
207                else:
208                    invalue = None
209            elif line.startswith('SIGGEN_LOCKEDSIGS_t-'):
210                invalue = line[18:].split('=', 1)[0].rstrip()
211                merged[invalue] = []
212                arch_order.append(invalue)
213
214    with open(lockedsigs_extra, 'r') as f:
215        invalue = None
216        tocopy = {}
217        for line in f:
218            if invalue:
219                if line.endswith('\\\n'):
220                    if not line in merged[invalue]:
221                        target, task = line.strip().split(':')[:2]
222                        if not copy_tasks or task in copy_tasks:
223                            tocopy[invalue].append(line)
224                        merged[invalue].append(line)
225                else:
226                    invalue = None
227            elif line.startswith('SIGGEN_LOCKEDSIGS_t-'):
228                invalue = line[18:].split('=', 1)[0].rstrip()
229                if not invalue in merged:
230                    merged[invalue] = []
231                    arch_order.append(invalue)
232                tocopy[invalue] = []
233
234    def write_sigs_file(fn, types, sigs):
235        fulltypes = []
236        bb.utils.mkdirhier(os.path.dirname(fn))
237        with open(fn, 'w') as f:
238            for typename in types:
239                lines = sigs[typename]
240                if lines:
241                    f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % typename)
242                    for line in lines:
243                        f.write(line)
244                    f.write('    "\n')
245                    fulltypes.append(typename)
246            f.write('SIGGEN_LOCKEDSIGS_TYPES = "%s"\n' % ' '.join(fulltypes))
247
248    if copy_output:
249        write_sigs_file(copy_output, list(tocopy.keys()), tocopy)
250    if merged_output:
251        write_sigs_file(merged_output, arch_order, merged)
252
253def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring="", filterfile=None):
254    import shutil
255    bb.note('Generating sstate-cache...')
256
257    nativelsbstring = d.getVar('NATIVELSBSTRING')
258    bb.process.run("PYTHONDONTWRITEBYTECODE=1 gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or ''))
259    if fixedlsbstring and nativelsbstring != fixedlsbstring:
260        nativedir = output_sstate_cache + '/' + nativelsbstring
261        if os.path.isdir(nativedir):
262            destdir = os.path.join(output_sstate_cache, fixedlsbstring)
263            for root, _, files in os.walk(nativedir):
264                for fn in files:
265                    src = os.path.join(root, fn)
266                    dest = os.path.join(destdir, os.path.relpath(src, nativedir))
267                    if os.path.exists(dest):
268                        # Already exists, and it'll be the same file, so just delete it
269                        os.unlink(src)
270                    else:
271                        bb.utils.mkdirhier(os.path.dirname(dest))
272                        shutil.move(src, dest)
273
274def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, logfile=None):
275    import subprocess
276
277    bb.note('Generating sstate task list...')
278
279    if not cwd:
280        cwd = os.getcwd()
281    if logfile:
282        logparam = '-l %s' % logfile
283    else:
284        logparam = ''
285    cmd = "%sPYTHONDONTWRITEBYTECODE=1 BB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
286    env = dict(d.getVar('BB_ORIGENV', False))
287    env.pop('BUILDDIR', '')
288    env.pop('BBPATH', '')
289    pathitems = env['PATH'].split(':')
290    env['PATH'] = ':'.join([item for item in pathitems if not item.endswith('/bitbake/bin')])
291    bb.process.run(cmd, stderr=subprocess.STDOUT, env=env, cwd=cwd, executable='/bin/bash')
292