1# Recipe creation tool - append plugin
2#
3# Copyright (C) 2015 Intel Corporation
4#
5# SPDX-License-Identifier: GPL-2.0-only
6#
7
8import sys
9import os
10import argparse
11import glob
12import fnmatch
13import re
14import subprocess
15import logging
16import stat
17import shutil
18import scriptutils
19import errno
20from collections import defaultdict
21import difflib
22
23logger = logging.getLogger('recipetool')
24
25tinfoil = None
26
27def tinfoil_init(instance):
28    global tinfoil
29    tinfoil = instance
30
31
32# FIXME guessing when we don't have pkgdata?
33# FIXME mode to create patch rather than directly substitute
34
35class InvalidTargetFileError(Exception):
36    pass
37
38def find_target_file(targetpath, d, pkglist=None):
39    """Find the recipe installing the specified target path, optionally limited to a select list of packages"""
40    import json
41
42    pkgdata_dir = d.getVar('PKGDATA_DIR')
43
44    # The mix between /etc and ${sysconfdir} here may look odd, but it is just
45    # being consistent with usage elsewhere
46    invalidtargets = {'${sysconfdir}/version': '${sysconfdir}/version is written out at image creation time',
47                      '/etc/timestamp': '/etc/timestamp is written out at image creation time',
48                      '/dev/*': '/dev is handled by udev (or equivalent) and the kernel (devtmpfs)',
49                      '/etc/passwd': '/etc/passwd should be managed through the useradd and extrausers classes',
50                      '/etc/group': '/etc/group should be managed through the useradd and extrausers classes',
51                      '/etc/shadow': '/etc/shadow should be managed through the useradd and extrausers classes',
52                      '/etc/gshadow': '/etc/gshadow should be managed through the useradd and extrausers classes',
53                      '${sysconfdir}/hostname': '${sysconfdir}/hostname contents should be set by setting hostname:pn-base-files = "value" in configuration',}
54
55    for pthspec, message in invalidtargets.items():
56        if fnmatch.fnmatchcase(targetpath, d.expand(pthspec)):
57            raise InvalidTargetFileError(d.expand(message))
58
59    targetpath_re = re.compile(r'\s+(\$D)?%s(\s|$)' % targetpath)
60
61    recipes = defaultdict(list)
62    for root, dirs, files in os.walk(os.path.join(pkgdata_dir, 'runtime')):
63        if pkglist:
64            filelist = pkglist
65        else:
66            filelist = files
67        for fn in filelist:
68            pkgdatafile = os.path.join(root, fn)
69            if pkglist and not os.path.exists(pkgdatafile):
70                continue
71            with open(pkgdatafile, 'r') as f:
72                pn = ''
73                # This does assume that PN comes before other values, but that's a fairly safe assumption
74                for line in f:
75                    if line.startswith('PN:'):
76                        pn = line.split(': ', 1)[1].strip()
77                    elif line.startswith('FILES_INFO'):
78                        val = line.split(': ', 1)[1].strip()
79                        dictval = json.loads(val)
80                        for fullpth in dictval.keys():
81                            if fnmatch.fnmatchcase(fullpth, targetpath):
82                                recipes[targetpath].append(pn)
83                    elif line.startswith('pkg_preinst:') or line.startswith('pkg_postinst:'):
84                        scriptval = line.split(': ', 1)[1].strip().encode('utf-8').decode('unicode_escape')
85                        if 'update-alternatives --install %s ' % targetpath in scriptval:
86                            recipes[targetpath].append('?%s' % pn)
87                        elif targetpath_re.search(scriptval):
88                            recipes[targetpath].append('!%s' % pn)
89    return recipes
90
91def _parse_recipe(pn, tinfoil):
92    try:
93        rd = tinfoil.parse_recipe(pn)
94    except bb.providers.NoProvider as e:
95        logger.error(str(e))
96        return None
97    return rd
98
99def determine_file_source(targetpath, rd):
100    """Assuming we know a file came from a specific recipe, figure out exactly where it came from"""
101    import oe.recipeutils
102
103    # See if it's in do_install for the recipe
104    workdir = rd.getVar('WORKDIR')
105    src_uri = rd.getVar('SRC_URI')
106    srcfile = ''
107    modpatches = []
108    elements = check_do_install(rd, targetpath)
109    if elements:
110        logger.debug('do_install line:\n%s' % ' '.join(elements))
111        srcpath = get_source_path(elements)
112        logger.debug('source path: %s' % srcpath)
113        if not srcpath.startswith('/'):
114            # Handle non-absolute path
115            srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs').split()[-1], srcpath))
116        if srcpath.startswith(workdir):
117            # OK, now we have the source file name, look for it in SRC_URI
118            workdirfile = os.path.relpath(srcpath, workdir)
119            # FIXME this is where we ought to have some code in the fetcher, because this is naive
120            for item in src_uri.split():
121                localpath = bb.fetch2.localpath(item, rd)
122                # Source path specified in do_install might be a glob
123                if fnmatch.fnmatch(os.path.basename(localpath), workdirfile):
124                    srcfile = 'file://%s' % localpath
125                elif '/' in workdirfile:
126                    if item == 'file://%s' % workdirfile:
127                        srcfile = 'file://%s' % localpath
128
129        # Check patches
130        srcpatches = []
131        patchedfiles = oe.recipeutils.get_recipe_patched_files(rd)
132        for patch, filelist in patchedfiles.items():
133            for fileitem in filelist:
134                if fileitem[0] == srcpath:
135                    srcpatches.append((patch, fileitem[1]))
136        if srcpatches:
137            addpatch = None
138            for patch in srcpatches:
139                if patch[1] == 'A':
140                    addpatch = patch[0]
141                else:
142                    modpatches.append(patch[0])
143            if addpatch:
144                srcfile = 'patch://%s' % addpatch
145
146    return (srcfile, elements, modpatches)
147
148def get_source_path(cmdelements):
149    """Find the source path specified within a command"""
150    command = cmdelements[0]
151    if command in ['install', 'cp']:
152        helptext = subprocess.check_output('LC_ALL=C %s --help' % command, shell=True).decode('utf-8')
153        argopts = ''
154        argopt_line_re = re.compile('^-([a-zA-Z0-9]), --[a-z-]+=')
155        for line in helptext.splitlines():
156            line = line.lstrip()
157            res = argopt_line_re.search(line)
158            if res:
159                argopts += res.group(1)
160        if not argopts:
161            # Fallback
162            if command == 'install':
163                argopts = 'gmoSt'
164            elif command == 'cp':
165                argopts = 't'
166            else:
167                raise Exception('No fallback arguments for command %s' % command)
168
169        skipnext = False
170        for elem in cmdelements[1:-1]:
171            if elem.startswith('-'):
172                if len(elem) > 1 and elem[1] in argopts:
173                    skipnext = True
174                continue
175            if skipnext:
176                skipnext = False
177                continue
178            return elem
179    else:
180        raise Exception('get_source_path: no handling for command "%s"')
181
182def get_func_deps(func, d):
183    """Find the function dependencies of a shell function"""
184    deps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func))
185    deps |= set((d.getVarFlag(func, "vardeps") or "").split())
186    funcdeps = []
187    for dep in deps:
188        if d.getVarFlag(dep, 'func'):
189            funcdeps.append(dep)
190    return funcdeps
191
192def check_do_install(rd, targetpath):
193    """Look at do_install for a command that installs/copies the specified target path"""
194    instpath = os.path.abspath(os.path.join(rd.getVar('D'), targetpath.lstrip('/')))
195    do_install = rd.getVar('do_install')
196    # Handle where do_install calls other functions (somewhat crudely, but good enough for this purpose)
197    deps = get_func_deps('do_install', rd)
198    for dep in deps:
199        do_install = do_install.replace(dep, rd.getVar(dep))
200
201    # Look backwards through do_install as we want to catch where a later line (perhaps
202    # from a bbappend) is writing over the top
203    for line in reversed(do_install.splitlines()):
204        line = line.strip()
205        if (line.startswith('install ') and ' -m' in line) or line.startswith('cp '):
206            elements = line.split()
207            destpath = os.path.abspath(elements[-1])
208            if destpath == instpath:
209                return elements
210            elif destpath.rstrip('/') == os.path.dirname(instpath):
211                # FIXME this doesn't take recursive copy into account; unsure if it's practical to do so
212                srcpath = get_source_path(elements)
213                if fnmatch.fnmatchcase(os.path.basename(instpath), os.path.basename(srcpath)):
214                    return elements
215    return None
216
217
218def appendfile(args):
219    import oe.recipeutils
220
221    stdout = ''
222    try:
223        (stdout, _) = bb.process.run('LANG=C file -b %s' % args.newfile, shell=True)
224        if 'cannot open' in stdout:
225            raise bb.process.ExecutionError(stdout)
226    except bb.process.ExecutionError as err:
227        logger.debug('file command returned error: %s' % err)
228        stdout = ''
229    if stdout:
230        logger.debug('file command output: %s' % stdout.rstrip())
231        if ('executable' in stdout and not 'shell script' in stdout) or 'shared object' in stdout:
232            logger.warning('This file looks like it is a binary or otherwise the output of compilation. If it is, you should consider building it properly instead of substituting a binary file directly.')
233
234    if args.recipe:
235        recipes = {args.targetpath: [args.recipe],}
236    else:
237        try:
238            recipes = find_target_file(args.targetpath, tinfoil.config_data)
239        except InvalidTargetFileError as e:
240            logger.error('%s cannot be handled by this tool: %s' % (args.targetpath, e))
241            return 1
242        if not recipes:
243            logger.error('Unable to find any package producing path %s - this may be because the recipe packaging it has not been built yet' % args.targetpath)
244            return 1
245
246    alternative_pns = []
247    postinst_pns = []
248
249    selectpn = None
250    for targetpath, pnlist in recipes.items():
251        for pn in pnlist:
252            if pn.startswith('?'):
253                alternative_pns.append(pn[1:])
254            elif pn.startswith('!'):
255                postinst_pns.append(pn[1:])
256            elif selectpn:
257                # hit here with multilibs
258                continue
259            else:
260                selectpn = pn
261
262    if not selectpn and len(alternative_pns) == 1:
263        selectpn = alternative_pns[0]
264        logger.error('File %s is an alternative possibly provided by recipe %s but seemingly no other, selecting it by default - you should double check other recipes' % (args.targetpath, selectpn))
265
266    if selectpn:
267        logger.debug('Selecting recipe %s for file %s' % (selectpn, args.targetpath))
268        if postinst_pns:
269            logger.warning('%s be modified by postinstall scripts for the following recipes:\n  %s\nThis may or may not be an issue depending on what modifications these postinstall scripts make.' % (args.targetpath, '\n  '.join(postinst_pns)))
270        rd = _parse_recipe(selectpn, tinfoil)
271        if not rd:
272            # Error message already shown
273            return 1
274        sourcefile, instelements, modpatches = determine_file_source(args.targetpath, rd)
275        sourcepath = None
276        if sourcefile:
277            sourcetype, sourcepath = sourcefile.split('://', 1)
278            logger.debug('Original source file is %s (%s)' % (sourcepath, sourcetype))
279            if sourcetype == 'patch':
280                logger.warning('File %s is added by the patch %s - you may need to remove or replace this patch in order to replace the file.' % (args.targetpath, sourcepath))
281                sourcepath = None
282        else:
283            logger.debug('Unable to determine source file, proceeding anyway')
284        if modpatches:
285            logger.warning('File %s is modified by the following patches:\n  %s' % (args.targetpath, '\n  '.join(modpatches)))
286
287        if instelements and sourcepath:
288            install = None
289        else:
290            # Auto-determine permissions
291            # Check destination
292            binpaths = '${bindir}:${sbindir}:${base_bindir}:${base_sbindir}:${libexecdir}:${sysconfdir}/init.d'
293            perms = '0644'
294            if os.path.abspath(os.path.dirname(args.targetpath)) in rd.expand(binpaths).split(':'):
295                # File is going into a directory normally reserved for executables, so it should be executable
296                perms = '0755'
297            else:
298                # Check source
299                st = os.stat(args.newfile)
300                if st.st_mode & stat.S_IXUSR:
301                    perms = '0755'
302            install = {args.newfile: (args.targetpath, perms)}
303        if sourcepath:
304            sourcepath = os.path.basename(sourcepath)
305        oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: {'newname' : sourcepath}}, install, wildcardver=args.wildcard_version, machine=args.machine)
306        tinfoil.modified_files()
307        return 0
308    else:
309        if alternative_pns:
310            logger.error('File %s is an alternative possibly provided by the following recipes:\n  %s\nPlease select recipe with -r/--recipe' % (targetpath, '\n  '.join(alternative_pns)))
311        elif postinst_pns:
312            logger.error('File %s may be written out in a pre/postinstall script of the following recipes:\n  %s\nPlease select recipe with -r/--recipe' % (targetpath, '\n  '.join(postinst_pns)))
313        return 3
314
315
316def appendsrc(args, files, rd, extralines=None):
317    import oe.recipeutils
318
319    srcdir = rd.getVar('S')
320    workdir = rd.getVar('WORKDIR')
321
322    import bb.fetch
323    simplified = {}
324    src_uri = rd.getVar('SRC_URI').split()
325    for uri in src_uri:
326        if uri.endswith(';'):
327            uri = uri[:-1]
328        simple_uri = bb.fetch.URI(uri)
329        simple_uri.params = {}
330        simplified[str(simple_uri)] = uri
331
332    copyfiles = {}
333    extralines = extralines or []
334    params = []
335    for newfile, srcfile in files.items():
336        src_destdir = os.path.dirname(srcfile)
337        if not args.use_workdir:
338            if rd.getVar('S') == rd.getVar('STAGING_KERNEL_DIR'):
339                srcdir = os.path.join(workdir, 'git')
340                if not bb.data.inherits_class('kernel-yocto', rd):
341                    logger.warning('S == STAGING_KERNEL_DIR and non-kernel-yocto, unable to determine path to srcdir, defaulting to ${WORKDIR}/git')
342            src_destdir = os.path.join(os.path.relpath(srcdir, workdir), src_destdir)
343        src_destdir = os.path.normpath(src_destdir)
344
345        if src_destdir and src_destdir != '.':
346            params.append({'subdir': src_destdir})
347        else:
348            params.append({})
349
350        copyfiles[newfile] = {'newname' : os.path.basename(srcfile)}
351
352    dry_run_output = None
353    dry_run_outdir = None
354    if args.dry_run:
355        import tempfile
356        dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
357        dry_run_outdir = dry_run_output.name
358
359    appendfile, _ = oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines, params=params,
360                                                   redirect_output=dry_run_outdir, update_original_recipe=args.update_recipe)
361    if not appendfile:
362        return
363    if args.dry_run:
364        output = ''
365        appendfilename = os.path.basename(appendfile)
366        newappendfile = appendfile
367        if appendfile and os.path.exists(appendfile):
368            with open(appendfile, 'r') as f:
369                oldlines = f.readlines()
370        else:
371            appendfile = '/dev/null'
372            oldlines = []
373
374        with open(os.path.join(dry_run_outdir, appendfilename), 'r') as f:
375            newlines = f.readlines()
376        diff = difflib.unified_diff(oldlines, newlines, appendfile, newappendfile)
377        difflines = list(diff)
378        if difflines:
379            output += ''.join(difflines)
380        if output:
381            logger.info('Diff of changed files:\n%s' % output)
382        else:
383            logger.info('No changed files')
384    tinfoil.modified_files()
385
386def appendsrcfiles(parser, args):
387    recipedata = _parse_recipe(args.recipe, tinfoil)
388    if not recipedata:
389        parser.error('RECIPE must be a valid recipe name')
390
391    files = dict((f, os.path.join(args.destdir, os.path.basename(f)))
392                 for f in args.files)
393    return appendsrc(args, files, recipedata)
394
395
396def appendsrcfile(parser, args):
397    recipedata = _parse_recipe(args.recipe, tinfoil)
398    if not recipedata:
399        parser.error('RECIPE must be a valid recipe name')
400
401    if not args.destfile:
402        args.destfile = os.path.basename(args.file)
403    elif args.destfile.endswith('/'):
404        args.destfile = os.path.join(args.destfile, os.path.basename(args.file))
405
406    return appendsrc(args, {args.file: args.destfile}, recipedata)
407
408
409def layer(layerpath):
410    if not os.path.exists(os.path.join(layerpath, 'conf', 'layer.conf')):
411        raise argparse.ArgumentTypeError('{0!r} must be a path to a valid layer'.format(layerpath))
412    return layerpath
413
414
415def existing_path(filepath):
416    if not os.path.exists(filepath):
417        raise argparse.ArgumentTypeError('{0!r} must be an existing path'.format(filepath))
418    return filepath
419
420
421def existing_file(filepath):
422    filepath = existing_path(filepath)
423    if os.path.isdir(filepath):
424        raise argparse.ArgumentTypeError('{0!r} must be a file, not a directory'.format(filepath))
425    return filepath
426
427
428def destination_path(destpath):
429    if os.path.isabs(destpath):
430        raise argparse.ArgumentTypeError('{0!r} must be a relative path, not absolute'.format(destpath))
431    return destpath
432
433
434def target_path(targetpath):
435    if not os.path.isabs(targetpath):
436        raise argparse.ArgumentTypeError('{0!r} must be an absolute path, not relative'.format(targetpath))
437    return targetpath
438
439
440def register_commands(subparsers):
441    common = argparse.ArgumentParser(add_help=False)
442    common.add_argument('-m', '--machine', help='Make bbappend changes specific to a machine only', metavar='MACHINE')
443    common.add_argument('-w', '--wildcard-version', help='Use wildcard to make the bbappend apply to any recipe version', action='store_true')
444    common.add_argument('destlayer', metavar='DESTLAYER', help='Base directory of the destination layer to write the bbappend to', type=layer)
445
446    parser_appendfile = subparsers.add_parser('appendfile',
447                                              parents=[common],
448                                              help='Create/update a bbappend to replace a target file',
449                                              description='Creates a bbappend (or updates an existing one) to replace the specified file that appears in the target system, determining the recipe that packages the file and the required path and name for the bbappend automatically. Note that the ability to determine the recipe packaging a particular file depends upon the recipe\'s do_packagedata task having already run prior to running this command (which it will have when the recipe has been built successfully, which in turn will have happened if one or more of the recipe\'s packages is included in an image that has been built successfully).')
450    parser_appendfile.add_argument('targetpath', help='Path to the file to be replaced (as it would appear within the target image, e.g. /etc/motd)', type=target_path)
451    parser_appendfile.add_argument('newfile', help='Custom file to replace the target file with', type=existing_file)
452    parser_appendfile.add_argument('-r', '--recipe', help='Override recipe to apply to (default is to find which recipe already packages the file)')
453    parser_appendfile.set_defaults(func=appendfile, parserecipes=True)
454
455    common_src = argparse.ArgumentParser(add_help=False, parents=[common])
456    common_src.add_argument('-W', '--workdir', help='Unpack file into WORKDIR rather than S', dest='use_workdir', action='store_true')
457    common_src.add_argument('recipe', metavar='RECIPE', help='Override recipe to apply to')
458
459    parser = subparsers.add_parser('appendsrcfiles',
460                                   parents=[common_src],
461                                   help='Create/update a bbappend to add or replace source files',
462                                   description='Creates a bbappend (or updates an existing one) to add or replace the specified file in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify multiple files with a destination directory, so cannot specify the destination filename. See the `appendsrcfile` command for the other behavior.')
463    parser.add_argument('-D', '--destdir', help='Destination directory (relative to S or WORKDIR, defaults to ".")', default='', type=destination_path)
464    parser.add_argument('-u', '--update-recipe', help='Update recipe instead of creating (or updating) a bbapend file. DESTLAYER must contains the recipe to update', action='store_true')
465    parser.add_argument('-n', '--dry-run', help='Dry run mode', action='store_true')
466    parser.add_argument('files', nargs='+', metavar='FILE', help='File(s) to be added to the recipe sources (WORKDIR or S)', type=existing_path)
467    parser.set_defaults(func=lambda a: appendsrcfiles(parser, a), parserecipes=True)
468
469    parser = subparsers.add_parser('appendsrcfile',
470                                   parents=[common_src],
471                                   help='Create/update a bbappend to add or replace a source file',
472                                   description='Creates a bbappend (or updates an existing one) to add or replace the specified files in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify the destination filename, not just destination directory, but only works for one file. See the `appendsrcfiles` command for the other behavior.')
473    parser.add_argument('-u', '--update-recipe', help='Update recipe instead of creating (or updating) a bbapend file. DESTLAYER must contains the recipe to update', action='store_true')
474    parser.add_argument('-n', '--dry-run', help='Dry run mode', action='store_true')
475    parser.add_argument('file', metavar='FILE', help='File to be added to the recipe sources (WORKDIR or S)', type=existing_path)
476    parser.add_argument('destfile', metavar='DESTFILE', nargs='?', help='Destination path (relative to S or WORKDIR, optional)', type=destination_path)
477    parser.set_defaults(func=lambda a: appendsrcfile(parser, a), parserecipes=True)
478