1#
2# BitBake Test for codeparser.py
3#
4# Copyright (C) 2010 Chris Larson
5# Copyright (C) 2012 Richard Purdie
6#
7# SPDX-License-Identifier: GPL-2.0-only
8#
9
10import unittest
11import logging
12import bb
13
14logger = logging.getLogger('BitBake.TestCodeParser')
15
16# bb.data references bb.parse but can't directly import due to circular dependencies.
17# Hack around it for now :(
18import bb.parse
19import bb.data
20
21class ReferenceTest(unittest.TestCase):
22    def setUp(self):
23        self.d = bb.data.init()
24
25    def setEmptyVars(self, varlist):
26        for k in varlist:
27            self.d.setVar(k, "")
28
29    def setValues(self, values):
30        for k, v in values.items():
31            self.d.setVar(k, v)
32
33    def assertReferences(self, refs):
34        self.assertEqual(self.references, refs)
35
36    def assertExecs(self, execs):
37        self.assertEqual(self.execs, execs)
38
39    def assertContains(self, contains):
40        self.assertEqual(self.contains, contains)
41
42class VariableReferenceTest(ReferenceTest):
43
44    def parseExpression(self, exp):
45        parsedvar = self.d.expandWithRefs(exp, None)
46        self.references = parsedvar.references
47        self.execs = parsedvar.execs
48
49    def test_simple_reference(self):
50        self.setEmptyVars(["FOO"])
51        self.parseExpression("${FOO}")
52        self.assertReferences(set(["FOO"]))
53
54    def test_nested_reference(self):
55        self.setEmptyVars(["BAR"])
56        self.d.setVar("FOO", "BAR")
57        self.parseExpression("${${FOO}}")
58        self.assertReferences(set(["FOO", "BAR"]))
59
60    def test_python_reference(self):
61        self.setEmptyVars(["BAR"])
62        self.parseExpression("${@d.getVar('BAR') + 'foo'}")
63        self.assertReferences(set(["BAR"]))
64
65    def test_python_exec_reference(self):
66        self.parseExpression("${@eval('3 * 5')}")
67        self.assertReferences(set())
68        self.assertExecs(set(["eval"]))
69
70class ShellReferenceTest(ReferenceTest):
71
72    def parseExpression(self, exp):
73        parsedvar = self.d.expandWithRefs(exp, None)
74        parser = bb.codeparser.ShellParser("ParserTest", logger)
75        parser.parse_shell(parsedvar.value)
76
77        self.references = parsedvar.references
78        self.execs = parser.execs
79
80    def test_quotes_inside_assign(self):
81        self.parseExpression('foo=foo"bar"baz')
82        self.assertReferences(set([]))
83
84    def test_quotes_inside_arg(self):
85        self.parseExpression('sed s#"bar baz"#"alpha beta"#g')
86        self.assertExecs(set(["sed"]))
87
88    def test_arg_continuation(self):
89        self.parseExpression("sed -i -e s,foo,bar,g \\\n *.pc")
90        self.assertExecs(set(["sed"]))
91
92    def test_dollar_in_quoted(self):
93        self.parseExpression('sed -i -e "foo$" *.pc')
94        self.assertExecs(set(["sed"]))
95
96    def test_quotes_inside_arg_continuation(self):
97        self.setEmptyVars(["bindir", "D", "libdir"])
98        self.parseExpression("""
99sed -i -e s#"moc_location=.*$"#"moc_location=${bindir}/moc4"# \\
100-e s#"uic_location=.*$"#"uic_location=${bindir}/uic4"# \\
101${D}${libdir}/pkgconfig/*.pc
102""")
103        self.assertReferences(set(["bindir", "D", "libdir"]))
104
105    def test_assign_subshell_expansion(self):
106        self.parseExpression("foo=$(echo bar)")
107        self.assertExecs(set(["echo"]))
108
109    def test_shell_unexpanded(self):
110        self.setEmptyVars(["QT_BASE_NAME"])
111        self.parseExpression('echo "${QT_BASE_NAME}"')
112        self.assertExecs(set(["echo"]))
113        self.assertReferences(set(["QT_BASE_NAME"]))
114
115    def test_incomplete_varexp_single_quotes(self):
116        self.parseExpression("sed -i -e 's:IP{:I${:g' $pc")
117        self.assertExecs(set(["sed"]))
118
119    def test_parameter_expansion_modifiers(self):
120        # -,+ and : are also valid modifiers for parameter expansion, but are
121        # valid characters in bitbake variable names, so are not included here
122        for i in ('=', '?', '#', '%', '##', '%%'):
123            name = "foo%sbar" % i
124            self.parseExpression("${%s}" % name)
125            self.assertNotIn(name, self.references)
126
127    def test_until(self):
128        self.parseExpression("until false; do echo true; done")
129        self.assertExecs(set(["false", "echo"]))
130        self.assertReferences(set())
131
132    def test_case(self):
133        self.parseExpression("""
134case $foo in
135*)
136bar
137;;
138esac
139""")
140        self.assertExecs(set(["bar"]))
141        self.assertReferences(set())
142
143    def test_assign_exec(self):
144        self.parseExpression("a=b c='foo bar' alpha 1 2 3")
145        self.assertExecs(set(["alpha"]))
146
147    def test_redirect_to_file(self):
148        self.setEmptyVars(["foo"])
149        self.parseExpression("echo foo >${foo}/bar")
150        self.assertExecs(set(["echo"]))
151        self.assertReferences(set(["foo"]))
152
153    def test_heredoc(self):
154        self.setEmptyVars(["theta"])
155        self.parseExpression("""
156cat <<END
157alpha
158beta
159${theta}
160END
161""")
162        self.assertReferences(set(["theta"]))
163
164    def test_redirect_from_heredoc(self):
165        v = ["B", "SHADOW_MAILDIR", "SHADOW_MAILFILE", "SHADOW_UTMPDIR", "SHADOW_LOGDIR", "bindir"]
166        self.setEmptyVars(v)
167        self.parseExpression("""
168cat <<END >${B}/cachedpaths
169shadow_cv_maildir=${SHADOW_MAILDIR}
170shadow_cv_mailfile=${SHADOW_MAILFILE}
171shadow_cv_utmpdir=${SHADOW_UTMPDIR}
172shadow_cv_logdir=${SHADOW_LOGDIR}
173shadow_cv_passwd_dir=${bindir}
174END
175""")
176        self.assertReferences(set(v))
177        self.assertExecs(set(["cat"]))
178
179#    def test_incomplete_command_expansion(self):
180#        self.assertRaises(reftracker.ShellSyntaxError, reftracker.execs,
181#                          bbvalue.shparse("cp foo`", self.d), self.d)
182
183#    def test_rogue_dollarsign(self):
184#        self.setValues({"D" : "/tmp"})
185#        self.parseExpression("install -d ${D}$")
186#        self.assertReferences(set(["D"]))
187#        self.assertExecs(set(["install"]))
188
189
190class PythonReferenceTest(ReferenceTest):
191
192    def setUp(self):
193        self.d = bb.data.init()
194        if hasattr(bb.utils, "_context"):
195            self.context = bb.utils._context
196        else:
197            import builtins
198            self.context = builtins.__dict__
199
200    def parseExpression(self, exp):
201        parsedvar = self.d.expandWithRefs(exp, None)
202        parser = bb.codeparser.PythonParser("ParserTest", logger)
203        parser.parse_python(parsedvar.value)
204
205        self.references = parsedvar.references | parser.references
206        self.execs = parser.execs
207        self.contains = parser.contains
208
209    @staticmethod
210    def indent(value):
211        """Python Snippets have to be indented, python values don't have to
212be. These unit tests are testing snippets."""
213        return " " + value
214
215    def test_getvar_reference(self):
216        self.parseExpression("d.getVar('foo')")
217        self.assertReferences(set(["foo"]))
218        self.assertExecs(set())
219
220    def test_getvar_computed_reference(self):
221        self.parseExpression("d.getVar('f' + 'o' + 'o')")
222        self.assertReferences(set())
223        self.assertExecs(set())
224
225    def test_getvar_exec_reference(self):
226        self.parseExpression("eval('d.getVar(\"foo\")')")
227        self.assertReferences(set())
228        self.assertExecs(set(["eval"]))
229
230    def test_var_reference(self):
231        self.context["foo"] = lambda x: x
232        self.setEmptyVars(["FOO"])
233        self.parseExpression("foo('${FOO}')")
234        self.assertReferences(set(["FOO"]))
235        self.assertExecs(set(["foo"]))
236        del self.context["foo"]
237
238    def test_var_exec(self):
239        for etype in ("func", "task"):
240            self.d.setVar("do_something", "echo 'hi mom! ${FOO}'")
241            self.d.setVarFlag("do_something", etype, True)
242            self.parseExpression("bb.build.exec_func('do_something', d)")
243            self.assertReferences(set([]))
244            self.assertExecs(set(["do_something"]))
245
246    def test_function_reference(self):
247        self.context["testfunc"] = lambda msg: bb.msg.note(1, None, msg)
248        self.d.setVar("FOO", "Hello, World!")
249        self.parseExpression("testfunc('${FOO}')")
250        self.assertReferences(set(["FOO"]))
251        self.assertExecs(set(["testfunc"]))
252        del self.context["testfunc"]
253
254    def test_qualified_function_reference(self):
255        self.parseExpression("time.time()")
256        self.assertExecs(set(["time.time"]))
257
258    def test_qualified_function_reference_2(self):
259        self.parseExpression("os.path.dirname('/foo/bar')")
260        self.assertExecs(set(["os.path.dirname"]))
261
262    def test_qualified_function_reference_nested(self):
263        self.parseExpression("time.strftime('%Y%m%d',time.gmtime())")
264        self.assertExecs(set(["time.strftime", "time.gmtime"]))
265
266    def test_function_reference_chained(self):
267        self.context["testget"] = lambda: "\tstrip me "
268        self.parseExpression("testget().strip()")
269        self.assertExecs(set(["testget"]))
270        del self.context["testget"]
271
272    def test_contains(self):
273        self.parseExpression('bb.utils.contains("TESTVAR", "one", "true", "false", d)')
274        self.assertContains({'TESTVAR': {'one'}})
275
276    def test_contains_multi(self):
277        self.parseExpression('bb.utils.contains("TESTVAR", "one two", "true", "false", d)')
278        self.assertContains({'TESTVAR': {'one two'}})
279
280    def test_contains_any(self):
281        self.parseExpression('bb.utils.contains_any("TESTVAR", "hello", "true", "false", d)')
282        self.assertContains({'TESTVAR': {'hello'}})
283
284    def test_contains_any_multi(self):
285        self.parseExpression('bb.utils.contains_any("TESTVAR", "one two three", "true", "false", d)')
286        self.assertContains({'TESTVAR': {'one', 'two', 'three'}})
287
288    def test_contains_filter(self):
289        self.parseExpression('bb.utils.filter("TESTVAR", "hello there world", d)')
290        self.assertContains({'TESTVAR': {'hello', 'there', 'world'}})
291
292
293class DependencyReferenceTest(ReferenceTest):
294
295    pydata = """
296d.getVar('somevar')
297def test(d):
298    foo = 'bar %s' % 'foo'
299def test2(d):
300    d.getVar(foo)
301    d.getVar('bar', False)
302    test2(d)
303
304def a():
305    \"\"\"some
306    stuff
307    \"\"\"
308    return "heh"
309
310test(d)
311
312d.expand(d.getVar("something", False))
313d.expand("${inexpand} somethingelse")
314d.getVar(a(), False)
315"""
316
317    def test_python(self):
318        self.d.setVar("FOO", self.pydata)
319        self.setEmptyVars(["inexpand", "a", "test2", "test"])
320        self.d.setVarFlags("FOO", {
321            "func": True,
322            "python": True,
323            "lineno": 1,
324            "filename": "example.bb",
325        })
326
327        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
328
329        self.assertEqual(deps, set(["somevar", "bar", "something", "inexpand", "test", "test2", "a"]))
330
331
332    shelldata = """
333foo () {
334bar
335}
336{
337echo baz
338$(heh)
339eval `moo`
340}
341a=b
342c=d
343(
344true && false
345test -f foo
346testval=something
347$testval
348) || aiee
349! inverted
350echo ${somevar}
351
352case foo in
353bar)
354echo bar
355;;
356baz)
357echo baz
358;;
359foo*)
360echo foo
361;;
362esac
363"""
364
365    def test_shell(self):
366        execs = ["bar", "echo", "heh", "moo", "true", "aiee"]
367        self.d.setVar("somevar", "heh")
368        self.d.setVar("inverted", "echo inverted...")
369        self.d.setVarFlag("inverted", "func", True)
370        self.d.setVar("FOO", self.shelldata)
371        self.d.setVarFlags("FOO", {"func": True})
372        self.setEmptyVars(execs)
373
374        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
375
376        self.assertEqual(deps, set(["somevar", "inverted"] + execs))
377
378
379    def test_vardeps(self):
380        self.d.setVar("oe_libinstall", "echo test")
381        self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
382        self.d.setVarFlag("FOO", "vardeps", "oe_libinstall")
383
384        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
385
386        self.assertEqual(deps, set(["oe_libinstall"]))
387
388    def test_vardeps_expand(self):
389        self.d.setVar("oe_libinstall", "echo test")
390        self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
391        self.d.setVarFlag("FOO", "vardeps", "${@'oe_libinstall'}")
392
393        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
394
395        self.assertEqual(deps, set(["oe_libinstall"]))
396
397    def test_contains_vardeps(self):
398        expr = '${@bb.utils.filter("TESTVAR", "somevalue anothervalue", d)} \
399                ${@bb.utils.contains("TESTVAR", "testval testval2", "yetanothervalue", "", d)} \
400                ${@bb.utils.contains("TESTVAR", "testval2 testval3", "blah", "", d)} \
401                ${@bb.utils.contains_any("TESTVAR", "testval2 testval3", "lastone", "", d)}'
402        parsedvar = self.d.expandWithRefs(expr, None)
403        # Check contains
404        self.assertEqual(parsedvar.contains, {'TESTVAR': {'testval2 testval3', 'anothervalue', 'somevalue', 'testval testval2', 'testval2', 'testval3'}})
405        # Check dependencies
406        self.d.setVar('ANOTHERVAR', expr)
407        self.d.setVar('TESTVAR', 'anothervalue testval testval2')
408        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
409        self.assertEqual(sorted(values.splitlines()),
410                         sorted([expr,
411                          'TESTVAR{anothervalue} = Set',
412                          'TESTVAR{somevalue} = Unset',
413                          'TESTVAR{testval testval2} = Set',
414                          'TESTVAR{testval2 testval3} = Unset',
415                          'TESTVAR{testval2} = Set',
416                          'TESTVAR{testval3} = Unset'
417                          ]))
418        # Check final value
419        self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['anothervalue', 'yetanothervalue', 'lastone'])
420
421    def test_contains_vardeps_excluded(self):
422        # Check the ignored_vars option to build_dependencies is handled by contains functionality
423        varval = '${TESTVAR2} ${@bb.utils.filter("TESTVAR", "somevalue anothervalue", d)}'
424        self.d.setVar('ANOTHERVAR', varval)
425        self.d.setVar('TESTVAR', 'anothervalue testval testval2')
426        self.d.setVar('TESTVAR2', 'testval3')
427        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(["TESTVAR"]), self.d, self.d)
428        self.assertEqual(sorted(values.splitlines()), sorted([varval]))
429        self.assertEqual(deps, set(["TESTVAR2"]))
430        self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue'])
431
432        # Check the vardepsexclude flag is handled by contains functionality
433        self.d.setVarFlag('ANOTHERVAR', 'vardepsexclude', 'TESTVAR')
434        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
435        self.assertEqual(sorted(values.splitlines()), sorted([varval]))
436        self.assertEqual(deps, set(["TESTVAR2"]))
437        self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue'])
438
439    def test_contains_vardeps_override_operators(self):
440        # Check override operators handle dependencies correctly with the contains functionality
441        expr_plain = 'testval'
442        expr_prepend = '${@bb.utils.filter("TESTVAR1", "testval1", d)} '
443        expr_append = ' ${@bb.utils.filter("TESTVAR2", "testval2", d)}'
444        expr_remove = '${@bb.utils.contains("TESTVAR3", "no-testval", "testval", "", d)}'
445        # Check dependencies
446        self.d.setVar('ANOTHERVAR', expr_plain)
447        self.d.prependVar('ANOTHERVAR', expr_prepend)
448        self.d.appendVar('ANOTHERVAR', expr_append)
449        self.d.setVar('ANOTHERVAR:remove', expr_remove)
450        self.d.setVar('TESTVAR1', 'blah')
451        self.d.setVar('TESTVAR2', 'testval2')
452        self.d.setVar('TESTVAR3', 'no-testval')
453        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
454        self.assertEqual(sorted(values.splitlines()),
455                         sorted([
456                          expr_prepend + expr_plain + expr_append,
457                          '_remove of ' + expr_remove,
458                          'TESTVAR1{testval1} = Unset',
459                          'TESTVAR2{testval2} = Set',
460                          'TESTVAR3{no-testval} = Set',
461                          ]))
462        # Check final value
463        self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval2'])
464
465    #Currently no wildcard support
466    #def test_vardeps_wildcards(self):
467    #    self.d.setVar("oe_libinstall", "echo test")
468    #    self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
469    #    self.d.setVarFlag("FOO", "vardeps", "oe_*")
470    #    self.assertEqual(deps, set(["oe_libinstall"]))
471
472
473