1#
2# BitBake Test for codeparser.py
3#
4# Copyright (C) 2010 Chris Larson
5# Copyright (C) 2012 Richard Purdie
6#
7# SPDX-License-Identifier: GPL-2.0-only
8#
9
10import unittest
11import logging
12import bb
13
14logger = logging.getLogger('BitBake.TestCodeParser')
15
16# bb.data references bb.parse but can't directly import due to circular dependencies.
17# Hack around it for now :(
18import bb.parse
19import bb.data
20
21class ReferenceTest(unittest.TestCase):
22    def setUp(self):
23        self.d = bb.data.init()
24
25    def setEmptyVars(self, varlist):
26        for k in varlist:
27            self.d.setVar(k, "")
28
29    def setValues(self, values):
30        for k, v in values.items():
31            self.d.setVar(k, v)
32
33    def assertReferences(self, refs):
34        self.assertEqual(self.references, refs)
35
36    def assertExecs(self, execs):
37        self.assertEqual(self.execs, execs)
38
39    def assertContains(self, contains):
40        self.assertEqual(self.contains, contains)
41
42class VariableReferenceTest(ReferenceTest):
43
44    def parseExpression(self, exp):
45        parsedvar = self.d.expandWithRefs(exp, None)
46        self.references = parsedvar.references
47
48    def test_simple_reference(self):
49        self.setEmptyVars(["FOO"])
50        self.parseExpression("${FOO}")
51        self.assertReferences(set(["FOO"]))
52
53    def test_nested_reference(self):
54        self.setEmptyVars(["BAR"])
55        self.d.setVar("FOO", "BAR")
56        self.parseExpression("${${FOO}}")
57        self.assertReferences(set(["FOO", "BAR"]))
58
59    def test_python_reference(self):
60        self.setEmptyVars(["BAR"])
61        self.parseExpression("${@d.getVar('BAR') + 'foo'}")
62        self.assertReferences(set(["BAR"]))
63
64class ShellReferenceTest(ReferenceTest):
65
66    def parseExpression(self, exp):
67        parsedvar = self.d.expandWithRefs(exp, None)
68        parser = bb.codeparser.ShellParser("ParserTest", logger)
69        parser.parse_shell(parsedvar.value)
70
71        self.references = parsedvar.references
72        self.execs = parser.execs
73
74    def test_quotes_inside_assign(self):
75        self.parseExpression('foo=foo"bar"baz')
76        self.assertReferences(set([]))
77
78    def test_quotes_inside_arg(self):
79        self.parseExpression('sed s#"bar baz"#"alpha beta"#g')
80        self.assertExecs(set(["sed"]))
81
82    def test_arg_continuation(self):
83        self.parseExpression("sed -i -e s,foo,bar,g \\\n *.pc")
84        self.assertExecs(set(["sed"]))
85
86    def test_dollar_in_quoted(self):
87        self.parseExpression('sed -i -e "foo$" *.pc')
88        self.assertExecs(set(["sed"]))
89
90    def test_quotes_inside_arg_continuation(self):
91        self.setEmptyVars(["bindir", "D", "libdir"])
92        self.parseExpression("""
93sed -i -e s#"moc_location=.*$"#"moc_location=${bindir}/moc4"# \\
94-e s#"uic_location=.*$"#"uic_location=${bindir}/uic4"# \\
95${D}${libdir}/pkgconfig/*.pc
96""")
97        self.assertReferences(set(["bindir", "D", "libdir"]))
98
99    def test_assign_subshell_expansion(self):
100        self.parseExpression("foo=$(echo bar)")
101        self.assertExecs(set(["echo"]))
102
103    def test_shell_unexpanded(self):
104        self.setEmptyVars(["QT_BASE_NAME"])
105        self.parseExpression('echo "${QT_BASE_NAME}"')
106        self.assertExecs(set(["echo"]))
107        self.assertReferences(set(["QT_BASE_NAME"]))
108
109    def test_incomplete_varexp_single_quotes(self):
110        self.parseExpression("sed -i -e 's:IP{:I${:g' $pc")
111        self.assertExecs(set(["sed"]))
112
113    def test_parameter_expansion_modifiers(self):
114        # -,+ and : are also valid modifiers for parameter expansion, but are
115        # valid characters in bitbake variable names, so are not included here
116        for i in ('=', '?', '#', '%', '##', '%%'):
117            name = "foo%sbar" % i
118            self.parseExpression("${%s}" % name)
119            self.assertNotIn(name, self.references)
120
121    def test_until(self):
122        self.parseExpression("until false; do echo true; done")
123        self.assertExecs(set(["false", "echo"]))
124        self.assertReferences(set())
125
126    def test_case(self):
127        self.parseExpression("""
128case $foo in
129*)
130bar
131;;
132esac
133""")
134        self.assertExecs(set(["bar"]))
135        self.assertReferences(set())
136
137    def test_assign_exec(self):
138        self.parseExpression("a=b c='foo bar' alpha 1 2 3")
139        self.assertExecs(set(["alpha"]))
140
141    def test_redirect_to_file(self):
142        self.setEmptyVars(["foo"])
143        self.parseExpression("echo foo >${foo}/bar")
144        self.assertExecs(set(["echo"]))
145        self.assertReferences(set(["foo"]))
146
147    def test_heredoc(self):
148        self.setEmptyVars(["theta"])
149        self.parseExpression("""
150cat <<END
151alpha
152beta
153${theta}
154END
155""")
156        self.assertReferences(set(["theta"]))
157
158    def test_redirect_from_heredoc(self):
159        v = ["B", "SHADOW_MAILDIR", "SHADOW_MAILFILE", "SHADOW_UTMPDIR", "SHADOW_LOGDIR", "bindir"]
160        self.setEmptyVars(v)
161        self.parseExpression("""
162cat <<END >${B}/cachedpaths
163shadow_cv_maildir=${SHADOW_MAILDIR}
164shadow_cv_mailfile=${SHADOW_MAILFILE}
165shadow_cv_utmpdir=${SHADOW_UTMPDIR}
166shadow_cv_logdir=${SHADOW_LOGDIR}
167shadow_cv_passwd_dir=${bindir}
168END
169""")
170        self.assertReferences(set(v))
171        self.assertExecs(set(["cat"]))
172
173#    def test_incomplete_command_expansion(self):
174#        self.assertRaises(reftracker.ShellSyntaxError, reftracker.execs,
175#                          bbvalue.shparse("cp foo`", self.d), self.d)
176
177#    def test_rogue_dollarsign(self):
178#        self.setValues({"D" : "/tmp"})
179#        self.parseExpression("install -d ${D}$")
180#        self.assertReferences(set(["D"]))
181#        self.assertExecs(set(["install"]))
182
183
184class PythonReferenceTest(ReferenceTest):
185
186    def setUp(self):
187        self.d = bb.data.init()
188        if hasattr(bb.utils, "_context"):
189            self.context = bb.utils._context
190        else:
191            import builtins
192            self.context = builtins.__dict__
193
194    def parseExpression(self, exp):
195        parsedvar = self.d.expandWithRefs(exp, None)
196        parser = bb.codeparser.PythonParser("ParserTest", logger)
197        parser.parse_python(parsedvar.value)
198
199        self.references = parsedvar.references | parser.references
200        self.execs = parser.execs
201        self.contains = parser.contains
202
203    @staticmethod
204    def indent(value):
205        """Python Snippets have to be indented, python values don't have to
206be. These unit tests are testing snippets."""
207        return " " + value
208
209    def test_getvar_reference(self):
210        self.parseExpression("d.getVar('foo')")
211        self.assertReferences(set(["foo"]))
212        self.assertExecs(set())
213
214    def test_getvar_computed_reference(self):
215        self.parseExpression("d.getVar('f' + 'o' + 'o')")
216        self.assertReferences(set())
217        self.assertExecs(set())
218
219    def test_getvar_exec_reference(self):
220        self.parseExpression("eval('d.getVar(\"foo\")')")
221        self.assertReferences(set())
222        self.assertExecs(set(["eval"]))
223
224    def test_var_reference(self):
225        self.context["foo"] = lambda x: x
226        self.setEmptyVars(["FOO"])
227        self.parseExpression("foo('${FOO}')")
228        self.assertReferences(set(["FOO"]))
229        self.assertExecs(set(["foo"]))
230        del self.context["foo"]
231
232    def test_var_exec(self):
233        for etype in ("func", "task"):
234            self.d.setVar("do_something", "echo 'hi mom! ${FOO}'")
235            self.d.setVarFlag("do_something", etype, True)
236            self.parseExpression("bb.build.exec_func('do_something', d)")
237            self.assertReferences(set([]))
238            self.assertExecs(set(["do_something"]))
239
240    def test_function_reference(self):
241        self.context["testfunc"] = lambda msg: bb.msg.note(1, None, msg)
242        self.d.setVar("FOO", "Hello, World!")
243        self.parseExpression("testfunc('${FOO}')")
244        self.assertReferences(set(["FOO"]))
245        self.assertExecs(set(["testfunc"]))
246        del self.context["testfunc"]
247
248    def test_qualified_function_reference(self):
249        self.parseExpression("time.time()")
250        self.assertExecs(set(["time.time"]))
251
252    def test_qualified_function_reference_2(self):
253        self.parseExpression("os.path.dirname('/foo/bar')")
254        self.assertExecs(set(["os.path.dirname"]))
255
256    def test_qualified_function_reference_nested(self):
257        self.parseExpression("time.strftime('%Y%m%d',time.gmtime())")
258        self.assertExecs(set(["time.strftime", "time.gmtime"]))
259
260    def test_function_reference_chained(self):
261        self.context["testget"] = lambda: "\tstrip me "
262        self.parseExpression("testget().strip()")
263        self.assertExecs(set(["testget"]))
264        del self.context["testget"]
265
266    def test_contains(self):
267        self.parseExpression('bb.utils.contains("TESTVAR", "one", "true", "false", d)')
268        self.assertContains({'TESTVAR': {'one'}})
269
270    def test_contains_multi(self):
271        self.parseExpression('bb.utils.contains("TESTVAR", "one two", "true", "false", d)')
272        self.assertContains({'TESTVAR': {'one two'}})
273
274    def test_contains_any(self):
275        self.parseExpression('bb.utils.contains_any("TESTVAR", "hello", "true", "false", d)')
276        self.assertContains({'TESTVAR': {'hello'}})
277
278    def test_contains_any_multi(self):
279        self.parseExpression('bb.utils.contains_any("TESTVAR", "one two three", "true", "false", d)')
280        self.assertContains({'TESTVAR': {'one', 'two', 'three'}})
281
282    def test_contains_filter(self):
283        self.parseExpression('bb.utils.filter("TESTVAR", "hello there world", d)')
284        self.assertContains({'TESTVAR': {'hello', 'there', 'world'}})
285
286
287class DependencyReferenceTest(ReferenceTest):
288
289    pydata = """
290d.getVar('somevar')
291def test(d):
292    foo = 'bar %s' % 'foo'
293def test2(d):
294    d.getVar(foo)
295    d.getVar('bar', False)
296    test2(d)
297
298def a():
299    \"\"\"some
300    stuff
301    \"\"\"
302    return "heh"
303
304test(d)
305
306d.expand(d.getVar("something", False))
307d.expand("${inexpand} somethingelse")
308d.getVar(a(), False)
309"""
310
311    def test_python(self):
312        self.d.setVar("FOO", self.pydata)
313        self.setEmptyVars(["inexpand", "a", "test2", "test"])
314        self.d.setVarFlags("FOO", {
315            "func": True,
316            "python": True,
317            "lineno": 1,
318            "filename": "example.bb",
319        })
320
321        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), self.d)
322
323        self.assertEqual(deps, set(["somevar", "bar", "something", "inexpand", "test", "test2", "a"]))
324
325
326    shelldata = """
327foo () {
328bar
329}
330{
331echo baz
332$(heh)
333eval `moo`
334}
335a=b
336c=d
337(
338true && false
339test -f foo
340testval=something
341$testval
342) || aiee
343! inverted
344echo ${somevar}
345
346case foo in
347bar)
348echo bar
349;;
350baz)
351echo baz
352;;
353foo*)
354echo foo
355;;
356esac
357"""
358
359    def test_shell(self):
360        execs = ["bar", "echo", "heh", "moo", "true", "aiee"]
361        self.d.setVar("somevar", "heh")
362        self.d.setVar("inverted", "echo inverted...")
363        self.d.setVarFlag("inverted", "func", True)
364        self.d.setVar("FOO", self.shelldata)
365        self.d.setVarFlags("FOO", {"func": True})
366        self.setEmptyVars(execs)
367
368        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), self.d)
369
370        self.assertEqual(deps, set(["somevar", "inverted"] + execs))
371
372
373    def test_vardeps(self):
374        self.d.setVar("oe_libinstall", "echo test")
375        self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
376        self.d.setVarFlag("FOO", "vardeps", "oe_libinstall")
377
378        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), self.d)
379
380        self.assertEqual(deps, set(["oe_libinstall"]))
381
382    def test_vardeps_expand(self):
383        self.d.setVar("oe_libinstall", "echo test")
384        self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
385        self.d.setVarFlag("FOO", "vardeps", "${@'oe_libinstall'}")
386
387        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), self.d)
388
389        self.assertEqual(deps, set(["oe_libinstall"]))
390
391    def test_contains_vardeps(self):
392        expr = '${@bb.utils.filter("TESTVAR", "somevalue anothervalue", d)} \
393                ${@bb.utils.contains("TESTVAR", "testval testval2", "yetanothervalue", "", d)} \
394                ${@bb.utils.contains("TESTVAR", "testval2 testval3", "blah", "", d)} \
395                ${@bb.utils.contains_any("TESTVAR", "testval2 testval3", "lastone", "", d)}'
396        parsedvar = self.d.expandWithRefs(expr, None)
397        # Check contains
398        self.assertEqual(parsedvar.contains, {'TESTVAR': {'testval2 testval3', 'anothervalue', 'somevalue', 'testval testval2', 'testval2', 'testval3'}})
399        # Check dependencies
400        self.d.setVar('ANOTHERVAR', expr)
401        self.d.setVar('TESTVAR', 'anothervalue testval testval2')
402        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), self.d)
403        self.assertEqual(sorted(values.splitlines()),
404                         sorted([expr,
405                          'TESTVAR{anothervalue} = Set',
406                          'TESTVAR{somevalue} = Unset',
407                          'TESTVAR{testval testval2} = Set',
408                          'TESTVAR{testval2 testval3} = Unset',
409                          'TESTVAR{testval2} = Set',
410                          'TESTVAR{testval3} = Unset'
411                          ]))
412        # Check final value
413        self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['anothervalue', 'yetanothervalue', 'lastone'])
414
415    def test_contains_vardeps_excluded(self):
416        # Check the ignored_vars option to build_dependencies is handled by contains functionality
417        varval = '${TESTVAR2} ${@bb.utils.filter("TESTVAR", "somevalue anothervalue", d)}'
418        self.d.setVar('ANOTHERVAR', varval)
419        self.d.setVar('TESTVAR', 'anothervalue testval testval2')
420        self.d.setVar('TESTVAR2', 'testval3')
421        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(["TESTVAR"]), self.d)
422        self.assertEqual(sorted(values.splitlines()), sorted([varval]))
423        self.assertEqual(deps, set(["TESTVAR2"]))
424        self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue'])
425
426        # Check the vardepsexclude flag is handled by contains functionality
427        self.d.setVarFlag('ANOTHERVAR', 'vardepsexclude', 'TESTVAR')
428        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), self.d)
429        self.assertEqual(sorted(values.splitlines()), sorted([varval]))
430        self.assertEqual(deps, set(["TESTVAR2"]))
431        self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue'])
432
433    #Currently no wildcard support
434    #def test_vardeps_wildcards(self):
435    #    self.d.setVar("oe_libinstall", "echo test")
436    #    self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
437    #    self.d.setVarFlag("FOO", "vardeps", "oe_*")
438    #    self.assertEquals(deps, set(["oe_libinstall"]))
439
440
441