1# 2# BitBake Test for codeparser.py 3# 4# Copyright (C) 2010 Chris Larson 5# Copyright (C) 2012 Richard Purdie 6# 7# SPDX-License-Identifier: GPL-2.0-only 8# 9 10import unittest 11import logging 12import bb 13 14logger = logging.getLogger('BitBake.TestCodeParser') 15 16# bb.data references bb.parse but can't directly import due to circular dependencies. 17# Hack around it for now :( 18import bb.parse 19import bb.data 20 21class ReferenceTest(unittest.TestCase): 22 def setUp(self): 23 self.d = bb.data.init() 24 25 def setEmptyVars(self, varlist): 26 for k in varlist: 27 self.d.setVar(k, "") 28 29 def setValues(self, values): 30 for k, v in values.items(): 31 self.d.setVar(k, v) 32 33 def assertReferences(self, refs): 34 self.assertEqual(self.references, refs) 35 36 def assertExecs(self, execs): 37 self.assertEqual(self.execs, execs) 38 39 def assertContains(self, contains): 40 self.assertEqual(self.contains, contains) 41 42class VariableReferenceTest(ReferenceTest): 43 44 def parseExpression(self, exp): 45 parsedvar = self.d.expandWithRefs(exp, None) 46 self.references = parsedvar.references 47 self.execs = parsedvar.execs 48 49 def test_simple_reference(self): 50 self.setEmptyVars(["FOO"]) 51 self.parseExpression("${FOO}") 52 self.assertReferences(set(["FOO"])) 53 54 def test_nested_reference(self): 55 self.setEmptyVars(["BAR"]) 56 self.d.setVar("FOO", "BAR") 57 self.parseExpression("${${FOO}}") 58 self.assertReferences(set(["FOO", "BAR"])) 59 60 def test_python_reference(self): 61 self.setEmptyVars(["BAR"]) 62 self.parseExpression("${@d.getVar('BAR') + 'foo'}") 63 self.assertReferences(set(["BAR"])) 64 65 def test_python_exec_reference(self): 66 self.parseExpression("${@eval('3 * 5')}") 67 self.assertReferences(set()) 68 self.assertExecs(set(["eval"])) 69 70class ShellReferenceTest(ReferenceTest): 71 72 def parseExpression(self, exp): 73 parsedvar = self.d.expandWithRefs(exp, None) 74 parser = bb.codeparser.ShellParser("ParserTest", logger) 75 parser.parse_shell(parsedvar.value) 76 77 self.references = parsedvar.references 78 self.execs = parser.execs 79 80 def test_quotes_inside_assign(self): 81 self.parseExpression('foo=foo"bar"baz') 82 self.assertReferences(set([])) 83 84 def test_quotes_inside_arg(self): 85 self.parseExpression('sed s#"bar baz"#"alpha beta"#g') 86 self.assertExecs(set(["sed"])) 87 88 def test_arg_continuation(self): 89 self.parseExpression("sed -i -e s,foo,bar,g \\\n *.pc") 90 self.assertExecs(set(["sed"])) 91 92 def test_dollar_in_quoted(self): 93 self.parseExpression('sed -i -e "foo$" *.pc') 94 self.assertExecs(set(["sed"])) 95 96 def test_quotes_inside_arg_continuation(self): 97 self.setEmptyVars(["bindir", "D", "libdir"]) 98 self.parseExpression(""" 99sed -i -e s#"moc_location=.*$"#"moc_location=${bindir}/moc4"# \\ 100-e s#"uic_location=.*$"#"uic_location=${bindir}/uic4"# \\ 101${D}${libdir}/pkgconfig/*.pc 102""") 103 self.assertReferences(set(["bindir", "D", "libdir"])) 104 105 def test_assign_subshell_expansion(self): 106 self.parseExpression("foo=$(echo bar)") 107 self.assertExecs(set(["echo"])) 108 109 def test_assign_subshell_expansion_quotes(self): 110 self.parseExpression('foo="$(echo bar)"') 111 self.assertExecs(set(["echo"])) 112 113 def test_assign_subshell_expansion_nested(self): 114 self.parseExpression('foo="$(func1 "$(func2 bar$(func3))")"') 115 self.assertExecs(set(["func1", "func2", "func3"])) 116 117 def test_assign_subshell_expansion_multiple(self): 118 self.parseExpression('foo="$(func1 "$(func2)") $(func3)"') 119 self.assertExecs(set(["func1", "func2", "func3"])) 120 121 def test_assign_subshell_expansion_escaped_quotes(self): 122 self.parseExpression('foo="\\"fo\\"o$(func1)"') 123 self.assertExecs(set(["func1"])) 124 125 def test_assign_subshell_expansion_empty(self): 126 self.parseExpression('foo="bar$()foo"') 127 self.assertExecs(set()) 128 129 def test_assign_subshell_backticks(self): 130 self.parseExpression("foo=`echo bar`") 131 self.assertExecs(set(["echo"])) 132 133 def test_assign_subshell_backticks_quotes(self): 134 self.parseExpression('foo="`echo bar`"') 135 self.assertExecs(set(["echo"])) 136 137 def test_assign_subshell_backticks_multiple(self): 138 self.parseExpression('foo="`func1 bar` `func2`"') 139 self.assertExecs(set(["func1", "func2"])) 140 141 def test_assign_subshell_backticks_escaped_quotes(self): 142 self.parseExpression('foo="\\"fo\\"o`func1`"') 143 self.assertExecs(set(["func1"])) 144 145 def test_assign_subshell_backticks_empty(self): 146 self.parseExpression('foo="bar``foo"') 147 self.assertExecs(set()) 148 149 def test_shell_unexpanded(self): 150 self.setEmptyVars(["QT_BASE_NAME"]) 151 self.parseExpression('echo "${QT_BASE_NAME}"') 152 self.assertExecs(set(["echo"])) 153 self.assertReferences(set(["QT_BASE_NAME"])) 154 155 def test_incomplete_varexp_single_quotes(self): 156 self.parseExpression("sed -i -e 's:IP{:I${:g' $pc") 157 self.assertExecs(set(["sed"])) 158 159 def test_parameter_expansion_modifiers(self): 160 # -,+ and : are also valid modifiers for parameter expansion, but are 161 # valid characters in bitbake variable names, so are not included here 162 for i in ('=', '?', '#', '%', '##', '%%'): 163 name = "foo%sbar" % i 164 self.parseExpression("${%s}" % name) 165 self.assertNotIn(name, self.references) 166 167 def test_until(self): 168 self.parseExpression("until false; do echo true; done") 169 self.assertExecs(set(["false", "echo"])) 170 self.assertReferences(set()) 171 172 def test_case(self): 173 self.parseExpression(""" 174case $foo in 175*) 176bar 177;; 178esac 179""") 180 self.assertExecs(set(["bar"])) 181 self.assertReferences(set()) 182 183 def test_assign_exec(self): 184 self.parseExpression("a=b c='foo bar' alpha 1 2 3") 185 self.assertExecs(set(["alpha"])) 186 187 def test_redirect_to_file(self): 188 self.setEmptyVars(["foo"]) 189 self.parseExpression("echo foo >${foo}/bar") 190 self.assertExecs(set(["echo"])) 191 self.assertReferences(set(["foo"])) 192 193 def test_heredoc(self): 194 self.setEmptyVars(["theta"]) 195 self.parseExpression(""" 196cat <<END 197alpha 198beta 199${theta} 200END 201""") 202 self.assertReferences(set(["theta"])) 203 204 def test_redirect_from_heredoc(self): 205 v = ["B", "SHADOW_MAILDIR", "SHADOW_MAILFILE", "SHADOW_UTMPDIR", "SHADOW_LOGDIR", "bindir"] 206 self.setEmptyVars(v) 207 self.parseExpression(""" 208cat <<END >${B}/cachedpaths 209shadow_cv_maildir=${SHADOW_MAILDIR} 210shadow_cv_mailfile=${SHADOW_MAILFILE} 211shadow_cv_utmpdir=${SHADOW_UTMPDIR} 212shadow_cv_logdir=${SHADOW_LOGDIR} 213shadow_cv_passwd_dir=${bindir} 214END 215""") 216 self.assertReferences(set(v)) 217 self.assertExecs(set(["cat"])) 218 219# def test_incomplete_command_expansion(self): 220# self.assertRaises(reftracker.ShellSyntaxError, reftracker.execs, 221# bbvalue.shparse("cp foo`", self.d), self.d) 222 223# def test_rogue_dollarsign(self): 224# self.setValues({"D" : "/tmp"}) 225# self.parseExpression("install -d ${D}$") 226# self.assertReferences(set(["D"])) 227# self.assertExecs(set(["install"])) 228 229 230class PythonReferenceTest(ReferenceTest): 231 232 def setUp(self): 233 self.d = bb.data.init() 234 if hasattr(bb.utils, "_context"): 235 self.context = bb.utils._context 236 else: 237 import builtins 238 self.context = builtins.__dict__ 239 240 def parseExpression(self, exp): 241 parsedvar = self.d.expandWithRefs(exp, None) 242 parser = bb.codeparser.PythonParser("ParserTest", logger) 243 parser.parse_python(parsedvar.value) 244 245 self.references = parsedvar.references | parser.references 246 self.execs = parser.execs 247 self.contains = parser.contains 248 249 @staticmethod 250 def indent(value): 251 """Python Snippets have to be indented, python values don't have to 252be. These unit tests are testing snippets.""" 253 return " " + value 254 255 def test_getvar_reference(self): 256 self.parseExpression("d.getVar('foo')") 257 self.assertReferences(set(["foo"])) 258 self.assertExecs(set()) 259 260 def test_getvar_computed_reference(self): 261 self.parseExpression("d.getVar('f' + 'o' + 'o')") 262 self.assertReferences(set()) 263 self.assertExecs(set()) 264 265 def test_getvar_exec_reference(self): 266 self.parseExpression("eval('d.getVar(\"foo\")')") 267 self.assertReferences(set()) 268 self.assertExecs(set(["eval"])) 269 270 def test_var_reference(self): 271 self.context["foo"] = lambda x: x 272 self.setEmptyVars(["FOO"]) 273 self.parseExpression("foo('${FOO}')") 274 self.assertReferences(set(["FOO"])) 275 self.assertExecs(set(["foo"])) 276 del self.context["foo"] 277 278 def test_var_exec(self): 279 for etype in ("func", "task"): 280 self.d.setVar("do_something", "echo 'hi mom! ${FOO}'") 281 self.d.setVarFlag("do_something", etype, True) 282 self.parseExpression("bb.build.exec_func('do_something', d)") 283 self.assertReferences(set([])) 284 self.assertExecs(set(["do_something"])) 285 286 def test_function_reference(self): 287 self.context["testfunc"] = lambda msg: bb.msg.note(1, None, msg) 288 self.d.setVar("FOO", "Hello, World!") 289 self.parseExpression("testfunc('${FOO}')") 290 self.assertReferences(set(["FOO"])) 291 self.assertExecs(set(["testfunc"])) 292 del self.context["testfunc"] 293 294 def test_qualified_function_reference(self): 295 self.parseExpression("time.time()") 296 self.assertExecs(set(["time.time"])) 297 298 def test_qualified_function_reference_2(self): 299 self.parseExpression("os.path.dirname('/foo/bar')") 300 self.assertExecs(set(["os.path.dirname"])) 301 302 def test_qualified_function_reference_nested(self): 303 self.parseExpression("time.strftime('%Y%m%d',time.gmtime())") 304 self.assertExecs(set(["time.strftime", "time.gmtime"])) 305 306 def test_function_reference_chained(self): 307 self.context["testget"] = lambda: "\tstrip me " 308 self.parseExpression("testget().strip()") 309 self.assertExecs(set(["testget"])) 310 del self.context["testget"] 311 312 def test_contains(self): 313 self.parseExpression('bb.utils.contains("TESTVAR", "one", "true", "false", d)') 314 self.assertContains({'TESTVAR': {'one'}}) 315 316 def test_contains_multi(self): 317 self.parseExpression('bb.utils.contains("TESTVAR", "one two", "true", "false", d)') 318 self.assertContains({'TESTVAR': {'one two'}}) 319 320 def test_contains_any(self): 321 self.parseExpression('bb.utils.contains_any("TESTVAR", "hello", "true", "false", d)') 322 self.assertContains({'TESTVAR': {'hello'}}) 323 324 def test_contains_any_multi(self): 325 self.parseExpression('bb.utils.contains_any("TESTVAR", "one two three", "true", "false", d)') 326 self.assertContains({'TESTVAR': {'one', 'two', 'three'}}) 327 328 def test_contains_filter(self): 329 self.parseExpression('bb.utils.filter("TESTVAR", "hello there world", d)') 330 self.assertContains({'TESTVAR': {'hello', 'there', 'world'}}) 331 332 333class DependencyReferenceTest(ReferenceTest): 334 335 pydata = """ 336d.getVar('somevar') 337def test(d): 338 foo = 'bar %s' % 'foo' 339def test2(d): 340 d.getVar(foo) 341 d.getVar('bar', False) 342 test2(d) 343 344def a(): 345 \"\"\"some 346 stuff 347 \"\"\" 348 return "heh" 349 350test(d) 351 352d.expand(d.getVar("something", False)) 353d.expand("${inexpand} somethingelse") 354d.getVar(a(), False) 355""" 356 357 def test_python(self): 358 self.d.setVar("FOO", self.pydata) 359 self.setEmptyVars(["inexpand", "a", "test2", "test"]) 360 self.d.setVarFlags("FOO", { 361 "func": True, 362 "python": True, 363 "lineno": 1, 364 "filename": "example.bb", 365 }) 366 367 deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d) 368 369 self.assertEqual(deps, set(["somevar", "bar", "something", "inexpand", "test", "test2", "a"])) 370 371 372 shelldata = """ 373foo () { 374bar 375} 376{ 377echo baz 378$(heh) 379eval `moo` 380} 381a=b 382c=d 383( 384true && false 385test -f foo 386testval=something 387$testval 388) || aiee 389! inverted 390echo ${somevar} 391 392case foo in 393bar) 394echo bar 395;; 396baz) 397echo baz 398;; 399foo*) 400echo foo 401;; 402esac 403""" 404 405 def test_shell(self): 406 execs = ["bar", "echo", "heh", "moo", "true", "aiee"] 407 self.d.setVar("somevar", "heh") 408 self.d.setVar("inverted", "echo inverted...") 409 self.d.setVarFlag("inverted", "func", True) 410 self.d.setVar("FOO", self.shelldata) 411 self.d.setVarFlags("FOO", {"func": True}) 412 self.setEmptyVars(execs) 413 414 deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d) 415 416 self.assertEqual(deps, set(["somevar", "inverted"] + execs)) 417 418 419 def test_vardeps(self): 420 self.d.setVar("oe_libinstall", "echo test") 421 self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") 422 self.d.setVarFlag("FOO", "vardeps", "oe_libinstall") 423 424 deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d) 425 426 self.assertEqual(deps, set(["oe_libinstall"])) 427 428 def test_vardeps_expand(self): 429 self.d.setVar("oe_libinstall", "echo test") 430 self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") 431 self.d.setVarFlag("FOO", "vardeps", "${@'oe_libinstall'}") 432 433 deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d) 434 435 self.assertEqual(deps, set(["oe_libinstall"])) 436 437 def test_contains_vardeps(self): 438 expr = '${@bb.utils.filter("TESTVAR", "somevalue anothervalue", d)} \ 439 ${@bb.utils.contains("TESTVAR", "testval testval2", "yetanothervalue", "", d)} \ 440 ${@bb.utils.contains("TESTVAR", "testval2 testval3", "blah", "", d)} \ 441 ${@bb.utils.contains_any("TESTVAR", "testval2 testval3", "lastone", "", d)}' 442 parsedvar = self.d.expandWithRefs(expr, None) 443 # Check contains 444 self.assertEqual(parsedvar.contains, {'TESTVAR': {'testval2 testval3', 'anothervalue', 'somevalue', 'testval testval2', 'testval2', 'testval3'}}) 445 # Check dependencies 446 self.d.setVar('ANOTHERVAR', expr) 447 self.d.setVar('TESTVAR', 'anothervalue testval testval2') 448 deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d) 449 self.assertEqual(sorted(values.splitlines()), 450 sorted([expr, 451 'TESTVAR{anothervalue} = Set', 452 'TESTVAR{somevalue} = Unset', 453 'TESTVAR{testval testval2} = Set', 454 'TESTVAR{testval2 testval3} = Unset', 455 'TESTVAR{testval2} = Set', 456 'TESTVAR{testval3} = Unset' 457 ])) 458 # Check final value 459 self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['anothervalue', 'yetanothervalue', 'lastone']) 460 461 def test_contains_vardeps_excluded(self): 462 # Check the ignored_vars option to build_dependencies is handled by contains functionality 463 varval = '${TESTVAR2} ${@bb.utils.filter("TESTVAR", "somevalue anothervalue", d)}' 464 self.d.setVar('ANOTHERVAR', varval) 465 self.d.setVar('TESTVAR', 'anothervalue testval testval2') 466 self.d.setVar('TESTVAR2', 'testval3') 467 deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(["TESTVAR"]), self.d, self.d) 468 self.assertEqual(sorted(values.splitlines()), sorted([varval])) 469 self.assertEqual(deps, set(["TESTVAR2"])) 470 self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue']) 471 472 # Check the vardepsexclude flag is handled by contains functionality 473 self.d.setVarFlag('ANOTHERVAR', 'vardepsexclude', 'TESTVAR') 474 deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d) 475 self.assertEqual(sorted(values.splitlines()), sorted([varval])) 476 self.assertEqual(deps, set(["TESTVAR2"])) 477 self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue']) 478 479 def test_contains_vardeps_override_operators(self): 480 # Check override operators handle dependencies correctly with the contains functionality 481 expr_plain = 'testval' 482 expr_prepend = '${@bb.utils.filter("TESTVAR1", "testval1", d)} ' 483 expr_append = ' ${@bb.utils.filter("TESTVAR2", "testval2", d)}' 484 expr_remove = '${@bb.utils.contains("TESTVAR3", "no-testval", "testval", "", d)}' 485 # Check dependencies 486 self.d.setVar('ANOTHERVAR', expr_plain) 487 self.d.prependVar('ANOTHERVAR', expr_prepend) 488 self.d.appendVar('ANOTHERVAR', expr_append) 489 self.d.setVar('ANOTHERVAR:remove', expr_remove) 490 self.d.setVar('TESTVAR1', 'blah') 491 self.d.setVar('TESTVAR2', 'testval2') 492 self.d.setVar('TESTVAR3', 'no-testval') 493 deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d) 494 self.assertEqual(sorted(values.splitlines()), 495 sorted([ 496 expr_prepend + expr_plain + expr_append, 497 '_remove of ' + expr_remove, 498 'TESTVAR1{testval1} = Unset', 499 'TESTVAR2{testval2} = Set', 500 'TESTVAR3{no-testval} = Set', 501 ])) 502 # Check final value 503 self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval2']) 504 505 #Currently no wildcard support 506 #def test_vardeps_wildcards(self): 507 # self.d.setVar("oe_libinstall", "echo test") 508 # self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") 509 # self.d.setVarFlag("FOO", "vardeps", "oe_*") 510 # self.assertEqual(deps, set(["oe_libinstall"])) 511 512 513