1# This class should provide easy access to the different aspects of the
2# buildsystem such as layers, bitbake location, etc.
3#
4# SDK_LAYERS_EXCLUDE: Layers which will be excluded from SDK layers.
5# SDK_LAYERS_EXCLUDE_PATTERN: The simiar to SDK_LAYERS_EXCLUDE, this supports
6#                             python regular expression, use space as separator,
7#                              e.g.: ".*-downloads closed-.*"
8#
9
10import stat
11import shutil
12
13def _smart_copy(src, dest):
14    import subprocess
15    # smart_copy will choose the correct function depending on whether the
16    # source is a file or a directory.
17    mode = os.stat(src).st_mode
18    if stat.S_ISDIR(mode):
19        bb.utils.mkdirhier(dest)
20        cmd = "tar --exclude='.git' --xattrs --xattrs-include='*' -chf - -C %s -p . \
21        | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dest)
22        subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
23    else:
24        shutil.copyfile(src, dest)
25        shutil.copymode(src, dest)
26
27class BuildSystem(object):
28    def __init__(self, context, d):
29        self.d = d
30        self.context = context
31        self.layerdirs = [os.path.abspath(pth) for pth in d.getVar('BBLAYERS').split()]
32        self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE') or "").split()
33        self.layers_exclude_pattern = d.getVar('SDK_LAYERS_EXCLUDE_PATTERN')
34
35    def copy_bitbake_and_layers(self, destdir, workspace_name=None):
36        import re
37        # Copy in all metadata layers + bitbake (as repositories)
38        copied_corebase = None
39        layers_copied = []
40        bb.utils.mkdirhier(destdir)
41        layers = list(self.layerdirs)
42
43        corebase = os.path.abspath(self.d.getVar('COREBASE'))
44        layers.append(corebase)
45        # The bitbake build system uses the meta-skeleton layer as a layout
46        # for common recipies, e.g: the recipetool script to create kernel recipies
47        # Add the meta-skeleton layer to be included as part of the eSDK installation
48        layers.append(os.path.join(corebase, 'meta-skeleton'))
49
50        # Exclude layers
51        for layer_exclude in self.layers_exclude:
52            if layer_exclude in layers:
53                bb.note('Excluded %s from sdk layers since it is in SDK_LAYERS_EXCLUDE' % layer_exclude)
54                layers.remove(layer_exclude)
55
56        if self.layers_exclude_pattern:
57            layers_cp = layers[:]
58            for pattern in self.layers_exclude_pattern.split():
59                for layer in layers_cp:
60                    if re.match(pattern, layer):
61                        bb.note('Excluded %s from sdk layers since matched SDK_LAYERS_EXCLUDE_PATTERN' % layer)
62                        layers.remove(layer)
63
64        workspace_newname = workspace_name
65        if workspace_newname:
66            layernames = [os.path.basename(layer) for layer in layers]
67            extranum = 0
68            while workspace_newname in layernames:
69                extranum += 1
70                workspace_newname = '%s-%d' % (workspace_name, extranum)
71
72        corebase_files = self.d.getVar('COREBASE_FILES').split()
73        corebase_files = [corebase + '/' +x for x in corebase_files]
74        # Make sure bitbake goes in
75        bitbake_dir = bb.__file__.rsplit('/', 3)[0]
76        corebase_files.append(bitbake_dir)
77
78        for layer in layers:
79            layerconf = os.path.join(layer, 'conf', 'layer.conf')
80            layernewname = os.path.basename(layer)
81            workspace = False
82            if os.path.exists(layerconf):
83                with open(layerconf, 'r') as f:
84                    if f.readline().startswith("# ### workspace layer auto-generated by devtool ###"):
85                        if workspace_newname:
86                            layernewname = workspace_newname
87                            workspace = True
88                        else:
89                            bb.plain("NOTE: Excluding local workspace layer %s from %s" % (layer, self.context))
90                            continue
91
92            # If the layer was already under corebase, leave it there
93            # since layers such as meta have issues when moved.
94            layerdestpath = destdir
95            if corebase == os.path.dirname(layer):
96                layerdestpath += '/' + os.path.basename(corebase)
97            else:
98                layer_relative = os.path.basename(corebase) + '/' + os.path.relpath(layer, corebase)
99                if os.path.dirname(layer_relative) != layernewname:
100                    layerdestpath += '/' + os.path.dirname(layer_relative)
101
102            layerdestpath += '/' + layernewname
103
104            layer_relative = os.path.relpath(layerdestpath,
105                                             destdir)
106            # Treat corebase as special since it typically will contain
107            # build directories or other custom items.
108            if corebase == layer:
109                copied_corebase = layer_relative
110                bb.utils.mkdirhier(layerdestpath)
111                for f in corebase_files:
112                    f_basename = os.path.basename(f)
113                    destname = os.path.join(layerdestpath, f_basename)
114                    _smart_copy(f, destname)
115            else:
116                layers_copied.append(layer_relative)
117
118                if os.path.exists(os.path.join(layerdestpath, 'conf/layer.conf')):
119                    bb.note("Skipping layer %s, already handled" % layer)
120                else:
121                    _smart_copy(layer, layerdestpath)
122
123            if workspace:
124                # Make some adjustments original workspace layer
125                # Drop sources (recipe tasks will be locked, so we don't need them)
126                srcdir = os.path.join(layerdestpath, 'sources')
127                if os.path.isdir(srcdir):
128                    shutil.rmtree(srcdir)
129                # Drop all bbappends except the one for the image the SDK is being built for
130                # (because of externalsrc, the workspace bbappends will interfere with the
131                # locked signatures if present, and we don't need them anyway)
132                image_bbappend = os.path.splitext(os.path.basename(self.d.getVar('FILE')))[0] + '.bbappend'
133                appenddir = os.path.join(layerdestpath, 'appends')
134                if os.path.isdir(appenddir):
135                    for fn in os.listdir(appenddir):
136                        if fn == image_bbappend:
137                            continue
138                        else:
139                            os.remove(os.path.join(appenddir, fn))
140                # Drop README
141                readme = os.path.join(layerdestpath, 'README')
142                if os.path.exists(readme):
143                    os.remove(readme)
144                # Filter out comments in layer.conf and change layer name
145                layerconf = os.path.join(layerdestpath, 'conf', 'layer.conf')
146                with open(layerconf, 'r') as f:
147                    origlines = f.readlines()
148                with open(layerconf, 'w') as f:
149                    for line in origlines:
150                        if line.startswith('#'):
151                            continue
152                        line = line.replace('workspacelayer', workspace_newname)
153                        f.write(line)
154
155        # meta-skeleton layer is added as part of the build system
156        # but not as a layer included in the build, therefore it is
157        # not reported to the function caller.
158        for layer in layers_copied:
159            if layer.endswith('/meta-skeleton'):
160                layers_copied.remove(layer)
161                break
162
163        return copied_corebase, layers_copied
164
165def generate_locked_sigs(sigfile, d):
166    bb.utils.mkdirhier(os.path.dirname(sigfile))
167    depd = d.getVar('BB_TASKDEPDATA', False)
168    tasks = ['%s.%s' % (v[2], v[1]) for v in depd.values()]
169    bb.parse.siggen.dump_lockedsigs(sigfile, tasks)
170
171def prune_lockedsigs(excluded_tasks, excluded_targets, lockedsigs, pruned_output):
172    with open(lockedsigs, 'r') as infile:
173        bb.utils.mkdirhier(os.path.dirname(pruned_output))
174        with open(pruned_output, 'w') as f:
175            invalue = False
176            for line in infile:
177                if invalue:
178                    if line.endswith('\\\n'):
179                        splitval = line.strip().split(':')
180                        if not splitval[1] in excluded_tasks and not splitval[0] in excluded_targets:
181                            f.write(line)
182                    else:
183                        f.write(line)
184                        invalue = False
185                elif line.startswith('SIGGEN_LOCKEDSIGS'):
186                    invalue = True
187                    f.write(line)
188
189def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_output, copy_output=None):
190    merged = {}
191    arch_order = []
192    with open(lockedsigs_main, 'r') as f:
193        invalue = None
194        for line in f:
195            if invalue:
196                if line.endswith('\\\n'):
197                    merged[invalue].append(line)
198                else:
199                    invalue = None
200            elif line.startswith('SIGGEN_LOCKEDSIGS_t-'):
201                invalue = line[18:].split('=', 1)[0].rstrip()
202                merged[invalue] = []
203                arch_order.append(invalue)
204
205    with open(lockedsigs_extra, 'r') as f:
206        invalue = None
207        tocopy = {}
208        for line in f:
209            if invalue:
210                if line.endswith('\\\n'):
211                    if not line in merged[invalue]:
212                        target, task = line.strip().split(':')[:2]
213                        if not copy_tasks or task in copy_tasks:
214                            tocopy[invalue].append(line)
215                        merged[invalue].append(line)
216                else:
217                    invalue = None
218            elif line.startswith('SIGGEN_LOCKEDSIGS_t-'):
219                invalue = line[18:].split('=', 1)[0].rstrip()
220                if not invalue in merged:
221                    merged[invalue] = []
222                    arch_order.append(invalue)
223                tocopy[invalue] = []
224
225    def write_sigs_file(fn, types, sigs):
226        fulltypes = []
227        bb.utils.mkdirhier(os.path.dirname(fn))
228        with open(fn, 'w') as f:
229            for typename in types:
230                lines = sigs[typename]
231                if lines:
232                    f.write('SIGGEN_LOCKEDSIGS_%s = "\\\n' % typename)
233                    for line in lines:
234                        f.write(line)
235                    f.write('    "\n')
236                    fulltypes.append(typename)
237            f.write('SIGGEN_LOCKEDSIGS_TYPES = "%s"\n' % ' '.join(fulltypes))
238
239    if copy_output:
240        write_sigs_file(copy_output, list(tocopy.keys()), tocopy)
241    if merged_output:
242        write_sigs_file(merged_output, arch_order, merged)
243
244def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring="", filterfile=None):
245    import shutil
246    bb.note('Generating sstate-cache...')
247
248    nativelsbstring = d.getVar('NATIVELSBSTRING')
249    bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or ''))
250    if fixedlsbstring and nativelsbstring != fixedlsbstring:
251        nativedir = output_sstate_cache + '/' + nativelsbstring
252        if os.path.isdir(nativedir):
253            destdir = os.path.join(output_sstate_cache, fixedlsbstring)
254            for root, _, files in os.walk(nativedir):
255                for fn in files:
256                    src = os.path.join(root, fn)
257                    dest = os.path.join(destdir, os.path.relpath(src, nativedir))
258                    if os.path.exists(dest):
259                        # Already exists, and it'll be the same file, so just delete it
260                        os.unlink(src)
261                    else:
262                        bb.utils.mkdirhier(os.path.dirname(dest))
263                        shutil.move(src, dest)
264
265def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, logfile=None):
266    import subprocess
267
268    bb.note('Generating sstate task list...')
269
270    if not cwd:
271        cwd = os.getcwd()
272    if logfile:
273        logparam = '-l %s' % logfile
274    else:
275        logparam = ''
276    cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
277    env = dict(d.getVar('BB_ORIGENV', False))
278    env.pop('BUILDDIR', '')
279    env.pop('BBPATH', '')
280    pathitems = env['PATH'].split(':')
281    env['PATH'] = ':'.join([item for item in pathitems if not item.endswith('/bitbake/bin')])
282    bb.process.run(cmd, stderr=subprocess.STDOUT, env=env, cwd=cwd, executable='/bin/bash')
283