1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3#
4# BitBake Test for codeparser.py
5#
6# Copyright (C) 2010 Chris Larson
7# Copyright (C) 2012 Richard Purdie
8#
9# This program is free software; you can redistribute it and/or modify
10# it under the terms of the GNU General Public License version 2 as
11# published by the Free Software Foundation.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the GNU General Public License along
19# with this program; if not, write to the Free Software Foundation, Inc.,
20# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21#
22
23import unittest
24import logging
25import bb
26
27logger = logging.getLogger('BitBake.TestCodeParser')
28
29# bb.data references bb.parse but can't directly import due to circular dependencies.
30# Hack around it for now :(
31import bb.parse
32import bb.data
33
34class ReferenceTest(unittest.TestCase):
35    def setUp(self):
36        self.d = bb.data.init()
37
38    def setEmptyVars(self, varlist):
39        for k in varlist:
40            self.d.setVar(k, "")
41
42    def setValues(self, values):
43        for k, v in values.items():
44            self.d.setVar(k, v)
45
46    def assertReferences(self, refs):
47        self.assertEqual(self.references, refs)
48
49    def assertExecs(self, execs):
50        self.assertEqual(self.execs, execs)
51
52    def assertContains(self, contains):
53        self.assertEqual(self.contains, contains)
54
55class VariableReferenceTest(ReferenceTest):
56
57    def parseExpression(self, exp):
58        parsedvar = self.d.expandWithRefs(exp, None)
59        self.references = parsedvar.references
60
61    def test_simple_reference(self):
62        self.setEmptyVars(["FOO"])
63        self.parseExpression("${FOO}")
64        self.assertReferences(set(["FOO"]))
65
66    def test_nested_reference(self):
67        self.setEmptyVars(["BAR"])
68        self.d.setVar("FOO", "BAR")
69        self.parseExpression("${${FOO}}")
70        self.assertReferences(set(["FOO", "BAR"]))
71
72    def test_python_reference(self):
73        self.setEmptyVars(["BAR"])
74        self.parseExpression("${@d.getVar('BAR') + 'foo'}")
75        self.assertReferences(set(["BAR"]))
76
77class ShellReferenceTest(ReferenceTest):
78
79    def parseExpression(self, exp):
80        parsedvar = self.d.expandWithRefs(exp, None)
81        parser = bb.codeparser.ShellParser("ParserTest", logger)
82        parser.parse_shell(parsedvar.value)
83
84        self.references = parsedvar.references
85        self.execs = parser.execs
86
87    def test_quotes_inside_assign(self):
88        self.parseExpression('foo=foo"bar"baz')
89        self.assertReferences(set([]))
90
91    def test_quotes_inside_arg(self):
92        self.parseExpression('sed s#"bar baz"#"alpha beta"#g')
93        self.assertExecs(set(["sed"]))
94
95    def test_arg_continuation(self):
96        self.parseExpression("sed -i -e s,foo,bar,g \\\n *.pc")
97        self.assertExecs(set(["sed"]))
98
99    def test_dollar_in_quoted(self):
100        self.parseExpression('sed -i -e "foo$" *.pc')
101        self.assertExecs(set(["sed"]))
102
103    def test_quotes_inside_arg_continuation(self):
104        self.setEmptyVars(["bindir", "D", "libdir"])
105        self.parseExpression("""
106sed -i -e s#"moc_location=.*$"#"moc_location=${bindir}/moc4"# \\
107-e s#"uic_location=.*$"#"uic_location=${bindir}/uic4"# \\
108${D}${libdir}/pkgconfig/*.pc
109""")
110        self.assertReferences(set(["bindir", "D", "libdir"]))
111
112    def test_assign_subshell_expansion(self):
113        self.parseExpression("foo=$(echo bar)")
114        self.assertExecs(set(["echo"]))
115
116    def test_shell_unexpanded(self):
117        self.setEmptyVars(["QT_BASE_NAME"])
118        self.parseExpression('echo "${QT_BASE_NAME}"')
119        self.assertExecs(set(["echo"]))
120        self.assertReferences(set(["QT_BASE_NAME"]))
121
122    def test_incomplete_varexp_single_quotes(self):
123        self.parseExpression("sed -i -e 's:IP{:I${:g' $pc")
124        self.assertExecs(set(["sed"]))
125
126    def test_parameter_expansion_modifiers(self):
127        # - and + are also valid modifiers for parameter expansion, but are
128        # valid characters in bitbake variable names, so are not included here
129        for i in ('=', ':-', ':=', '?', ':?', ':+', '#', '%', '##', '%%'):
130            name = "foo%sbar" % i
131            self.parseExpression("${%s}" % name)
132            self.assertNotIn(name, self.references)
133
134    def test_until(self):
135        self.parseExpression("until false; do echo true; done")
136        self.assertExecs(set(["false", "echo"]))
137        self.assertReferences(set())
138
139    def test_case(self):
140        self.parseExpression("""
141case $foo in
142*)
143bar
144;;
145esac
146""")
147        self.assertExecs(set(["bar"]))
148        self.assertReferences(set())
149
150    def test_assign_exec(self):
151        self.parseExpression("a=b c='foo bar' alpha 1 2 3")
152        self.assertExecs(set(["alpha"]))
153
154    def test_redirect_to_file(self):
155        self.setEmptyVars(["foo"])
156        self.parseExpression("echo foo >${foo}/bar")
157        self.assertExecs(set(["echo"]))
158        self.assertReferences(set(["foo"]))
159
160    def test_heredoc(self):
161        self.setEmptyVars(["theta"])
162        self.parseExpression("""
163cat <<END
164alpha
165beta
166${theta}
167END
168""")
169        self.assertReferences(set(["theta"]))
170
171    def test_redirect_from_heredoc(self):
172        v = ["B", "SHADOW_MAILDIR", "SHADOW_MAILFILE", "SHADOW_UTMPDIR", "SHADOW_LOGDIR", "bindir"]
173        self.setEmptyVars(v)
174        self.parseExpression("""
175cat <<END >${B}/cachedpaths
176shadow_cv_maildir=${SHADOW_MAILDIR}
177shadow_cv_mailfile=${SHADOW_MAILFILE}
178shadow_cv_utmpdir=${SHADOW_UTMPDIR}
179shadow_cv_logdir=${SHADOW_LOGDIR}
180shadow_cv_passwd_dir=${bindir}
181END
182""")
183        self.assertReferences(set(v))
184        self.assertExecs(set(["cat"]))
185
186#    def test_incomplete_command_expansion(self):
187#        self.assertRaises(reftracker.ShellSyntaxError, reftracker.execs,
188#                          bbvalue.shparse("cp foo`", self.d), self.d)
189
190#    def test_rogue_dollarsign(self):
191#        self.setValues({"D" : "/tmp"})
192#        self.parseExpression("install -d ${D}$")
193#        self.assertReferences(set(["D"]))
194#        self.assertExecs(set(["install"]))
195
196
197class PythonReferenceTest(ReferenceTest):
198
199    def setUp(self):
200        self.d = bb.data.init()
201        if hasattr(bb.utils, "_context"):
202            self.context = bb.utils._context
203        else:
204            import builtins
205            self.context = builtins.__dict__
206
207    def parseExpression(self, exp):
208        parsedvar = self.d.expandWithRefs(exp, None)
209        parser = bb.codeparser.PythonParser("ParserTest", logger)
210        parser.parse_python(parsedvar.value)
211
212        self.references = parsedvar.references | parser.references
213        self.execs = parser.execs
214        self.contains = parser.contains
215
216    @staticmethod
217    def indent(value):
218        """Python Snippets have to be indented, python values don't have to
219be. These unit tests are testing snippets."""
220        return " " + value
221
222    def test_getvar_reference(self):
223        self.parseExpression("d.getVar('foo')")
224        self.assertReferences(set(["foo"]))
225        self.assertExecs(set())
226
227    def test_getvar_computed_reference(self):
228        self.parseExpression("d.getVar('f' + 'o' + 'o')")
229        self.assertReferences(set())
230        self.assertExecs(set())
231
232    def test_getvar_exec_reference(self):
233        self.parseExpression("eval('d.getVar(\"foo\")')")
234        self.assertReferences(set())
235        self.assertExecs(set(["eval"]))
236
237    def test_var_reference(self):
238        self.context["foo"] = lambda x: x
239        self.setEmptyVars(["FOO"])
240        self.parseExpression("foo('${FOO}')")
241        self.assertReferences(set(["FOO"]))
242        self.assertExecs(set(["foo"]))
243        del self.context["foo"]
244
245    def test_var_exec(self):
246        for etype in ("func", "task"):
247            self.d.setVar("do_something", "echo 'hi mom! ${FOO}'")
248            self.d.setVarFlag("do_something", etype, True)
249            self.parseExpression("bb.build.exec_func('do_something', d)")
250            self.assertReferences(set([]))
251            self.assertExecs(set(["do_something"]))
252
253    def test_function_reference(self):
254        self.context["testfunc"] = lambda msg: bb.msg.note(1, None, msg)
255        self.d.setVar("FOO", "Hello, World!")
256        self.parseExpression("testfunc('${FOO}')")
257        self.assertReferences(set(["FOO"]))
258        self.assertExecs(set(["testfunc"]))
259        del self.context["testfunc"]
260
261    def test_qualified_function_reference(self):
262        self.parseExpression("time.time()")
263        self.assertExecs(set(["time.time"]))
264
265    def test_qualified_function_reference_2(self):
266        self.parseExpression("os.path.dirname('/foo/bar')")
267        self.assertExecs(set(["os.path.dirname"]))
268
269    def test_qualified_function_reference_nested(self):
270        self.parseExpression("time.strftime('%Y%m%d',time.gmtime())")
271        self.assertExecs(set(["time.strftime", "time.gmtime"]))
272
273    def test_function_reference_chained(self):
274        self.context["testget"] = lambda: "\tstrip me "
275        self.parseExpression("testget().strip()")
276        self.assertExecs(set(["testget"]))
277        del self.context["testget"]
278
279    def test_contains(self):
280        self.parseExpression('bb.utils.contains("TESTVAR", "one", "true", "false", d)')
281        self.assertContains({'TESTVAR': {'one'}})
282
283    def test_contains_multi(self):
284        self.parseExpression('bb.utils.contains("TESTVAR", "one two", "true", "false", d)')
285        self.assertContains({'TESTVAR': {'one two'}})
286
287    def test_contains_any(self):
288        self.parseExpression('bb.utils.contains_any("TESTVAR", "hello", "true", "false", d)')
289        self.assertContains({'TESTVAR': {'hello'}})
290
291    def test_contains_any_multi(self):
292        self.parseExpression('bb.utils.contains_any("TESTVAR", "one two three", "true", "false", d)')
293        self.assertContains({'TESTVAR': {'one', 'two', 'three'}})
294
295    def test_contains_filter(self):
296        self.parseExpression('bb.utils.filter("TESTVAR", "hello there world", d)')
297        self.assertContains({'TESTVAR': {'hello', 'there', 'world'}})
298
299
300class DependencyReferenceTest(ReferenceTest):
301
302    pydata = """
303d.getVar('somevar')
304def test(d):
305    foo = 'bar %s' % 'foo'
306def test2(d):
307    d.getVar(foo)
308    d.getVar('bar', False)
309    test2(d)
310
311def a():
312    \"\"\"some
313    stuff
314    \"\"\"
315    return "heh"
316
317test(d)
318
319d.expand(d.getVar("something", False))
320d.expand("${inexpand} somethingelse")
321d.getVar(a(), False)
322"""
323
324    def test_python(self):
325        self.d.setVar("FOO", self.pydata)
326        self.setEmptyVars(["inexpand", "a", "test2", "test"])
327        self.d.setVarFlags("FOO", {
328            "func": True,
329            "python": True,
330            "lineno": 1,
331            "filename": "example.bb",
332        })
333
334        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d)
335
336        self.assertEqual(deps, set(["somevar", "bar", "something", "inexpand", "test", "test2", "a"]))
337
338
339    shelldata = """
340foo () {
341bar
342}
343{
344echo baz
345$(heh)
346eval `moo`
347}
348a=b
349c=d
350(
351true && false
352test -f foo
353testval=something
354$testval
355) || aiee
356! inverted
357echo ${somevar}
358
359case foo in
360bar)
361echo bar
362;;
363baz)
364echo baz
365;;
366foo*)
367echo foo
368;;
369esac
370"""
371
372    def test_shell(self):
373        execs = ["bar", "echo", "heh", "moo", "true", "aiee"]
374        self.d.setVar("somevar", "heh")
375        self.d.setVar("inverted", "echo inverted...")
376        self.d.setVarFlag("inverted", "func", True)
377        self.d.setVar("FOO", self.shelldata)
378        self.d.setVarFlags("FOO", {"func": True})
379        self.setEmptyVars(execs)
380
381        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d)
382
383        self.assertEqual(deps, set(["somevar", "inverted"] + execs))
384
385
386    def test_vardeps(self):
387        self.d.setVar("oe_libinstall", "echo test")
388        self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
389        self.d.setVarFlag("FOO", "vardeps", "oe_libinstall")
390
391        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d)
392
393        self.assertEqual(deps, set(["oe_libinstall"]))
394
395    def test_vardeps_expand(self):
396        self.d.setVar("oe_libinstall", "echo test")
397        self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
398        self.d.setVarFlag("FOO", "vardeps", "${@'oe_libinstall'}")
399
400        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d)
401
402        self.assertEqual(deps, set(["oe_libinstall"]))
403
404    def test_contains_vardeps(self):
405        expr = '${@bb.utils.filter("TESTVAR", "somevalue anothervalue", d)} \
406                ${@bb.utils.contains("TESTVAR", "testval testval2", "yetanothervalue", "", d)} \
407                ${@bb.utils.contains("TESTVAR", "testval2 testval3", "blah", "", d)} \
408                ${@bb.utils.contains_any("TESTVAR", "testval2 testval3", "lastone", "", d)}'
409        parsedvar = self.d.expandWithRefs(expr, None)
410        # Check contains
411        self.assertEqual(parsedvar.contains, {'TESTVAR': {'testval2 testval3', 'anothervalue', 'somevalue', 'testval testval2', 'testval2', 'testval3'}})
412        # Check dependencies
413        self.d.setVar('ANOTHERVAR', expr)
414        self.d.setVar('TESTVAR', 'anothervalue testval testval2')
415        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), self.d)
416        self.assertEqual(sorted(values.splitlines()),
417                         sorted([expr,
418                          'TESTVAR{anothervalue} = Set',
419                          'TESTVAR{somevalue} = Unset',
420                          'TESTVAR{testval testval2} = Set',
421                          'TESTVAR{testval2 testval3} = Unset',
422                          'TESTVAR{testval2} = Set',
423                          'TESTVAR{testval3} = Unset'
424                          ]))
425        # Check final value
426        self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['anothervalue', 'yetanothervalue', 'lastone'])
427
428    #Currently no wildcard support
429    #def test_vardeps_wildcards(self):
430    #    self.d.setVar("oe_libinstall", "echo test")
431    #    self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
432    #    self.d.setVarFlag("FOO", "vardeps", "oe_*")
433    #    self.assertEquals(deps, set(["oe_libinstall"]))
434
435
436