1# 2# BitBake Test for codeparser.py 3# 4# Copyright (C) 2010 Chris Larson 5# Copyright (C) 2012 Richard Purdie 6# 7# SPDX-License-Identifier: GPL-2.0-only 8# 9 10import unittest 11import logging 12import bb 13 14logger = logging.getLogger('BitBake.TestCodeParser') 15 16# bb.data references bb.parse but can't directly import due to circular dependencies. 17# Hack around it for now :( 18import bb.parse 19import bb.data 20 21class ReferenceTest(unittest.TestCase): 22 def setUp(self): 23 self.d = bb.data.init() 24 25 def setEmptyVars(self, varlist): 26 for k in varlist: 27 self.d.setVar(k, "") 28 29 def setValues(self, values): 30 for k, v in values.items(): 31 self.d.setVar(k, v) 32 33 def assertReferences(self, refs): 34 self.assertEqual(self.references, refs) 35 36 def assertExecs(self, execs): 37 self.assertEqual(self.execs, execs) 38 39 def assertContains(self, contains): 40 self.assertEqual(self.contains, contains) 41 42class VariableReferenceTest(ReferenceTest): 43 44 def parseExpression(self, exp): 45 parsedvar = self.d.expandWithRefs(exp, None) 46 self.references = parsedvar.references 47 self.execs = parsedvar.execs 48 49 def test_simple_reference(self): 50 self.setEmptyVars(["FOO"]) 51 self.parseExpression("${FOO}") 52 self.assertReferences(set(["FOO"])) 53 54 def test_nested_reference(self): 55 self.setEmptyVars(["BAR"]) 56 self.d.setVar("FOO", "BAR") 57 self.parseExpression("${${FOO}}") 58 self.assertReferences(set(["FOO", "BAR"])) 59 60 def test_python_reference(self): 61 self.setEmptyVars(["BAR"]) 62 self.parseExpression("${@d.getVar('BAR') + 'foo'}") 63 self.assertReferences(set(["BAR"])) 64 65 def test_python_exec_reference(self): 66 self.parseExpression("${@eval('3 * 5')}") 67 self.assertReferences(set()) 68 self.assertExecs(set(["eval"])) 69 70class ShellReferenceTest(ReferenceTest): 71 72 def parseExpression(self, exp): 73 parsedvar = self.d.expandWithRefs(exp, None) 74 parser = bb.codeparser.ShellParser("ParserTest", logger) 75 parser.parse_shell(parsedvar.value) 76 77 self.references = parsedvar.references 78 self.execs = parser.execs 79 80 def test_quotes_inside_assign(self): 81 self.parseExpression('foo=foo"bar"baz') 82 self.assertReferences(set([])) 83 84 def test_quotes_inside_arg(self): 85 self.parseExpression('sed s#"bar baz"#"alpha beta"#g') 86 self.assertExecs(set(["sed"])) 87 88 def test_arg_continuation(self): 89 self.parseExpression("sed -i -e s,foo,bar,g \\\n *.pc") 90 self.assertExecs(set(["sed"])) 91 92 def test_dollar_in_quoted(self): 93 self.parseExpression('sed -i -e "foo$" *.pc') 94 self.assertExecs(set(["sed"])) 95 96 def test_quotes_inside_arg_continuation(self): 97 self.setEmptyVars(["bindir", "D", "libdir"]) 98 self.parseExpression(""" 99sed -i -e s#"moc_location=.*$"#"moc_location=${bindir}/moc4"# \\ 100-e s#"uic_location=.*$"#"uic_location=${bindir}/uic4"# \\ 101${D}${libdir}/pkgconfig/*.pc 102""") 103 self.assertReferences(set(["bindir", "D", "libdir"])) 104 105 def test_assign_subshell_expansion(self): 106 self.parseExpression("foo=$(echo bar)") 107 self.assertExecs(set(["echo"])) 108 109 def test_shell_unexpanded(self): 110 self.setEmptyVars(["QT_BASE_NAME"]) 111 self.parseExpression('echo "${QT_BASE_NAME}"') 112 self.assertExecs(set(["echo"])) 113 self.assertReferences(set(["QT_BASE_NAME"])) 114 115 def test_incomplete_varexp_single_quotes(self): 116 self.parseExpression("sed -i -e 's:IP{:I${:g' $pc") 117 self.assertExecs(set(["sed"])) 118 119 def test_parameter_expansion_modifiers(self): 120 # -,+ and : are also valid modifiers for parameter expansion, but are 121 # valid characters in bitbake variable names, so are not included here 122 for i in ('=', '?', '#', '%', '##', '%%'): 123 name = "foo%sbar" % i 124 self.parseExpression("${%s}" % name) 125 self.assertNotIn(name, self.references) 126 127 def test_until(self): 128 self.parseExpression("until false; do echo true; done") 129 self.assertExecs(set(["false", "echo"])) 130 self.assertReferences(set()) 131 132 def test_case(self): 133 self.parseExpression(""" 134case $foo in 135*) 136bar 137;; 138esac 139""") 140 self.assertExecs(set(["bar"])) 141 self.assertReferences(set()) 142 143 def test_assign_exec(self): 144 self.parseExpression("a=b c='foo bar' alpha 1 2 3") 145 self.assertExecs(set(["alpha"])) 146 147 def test_redirect_to_file(self): 148 self.setEmptyVars(["foo"]) 149 self.parseExpression("echo foo >${foo}/bar") 150 self.assertExecs(set(["echo"])) 151 self.assertReferences(set(["foo"])) 152 153 def test_heredoc(self): 154 self.setEmptyVars(["theta"]) 155 self.parseExpression(""" 156cat <<END 157alpha 158beta 159${theta} 160END 161""") 162 self.assertReferences(set(["theta"])) 163 164 def test_redirect_from_heredoc(self): 165 v = ["B", "SHADOW_MAILDIR", "SHADOW_MAILFILE", "SHADOW_UTMPDIR", "SHADOW_LOGDIR", "bindir"] 166 self.setEmptyVars(v) 167 self.parseExpression(""" 168cat <<END >${B}/cachedpaths 169shadow_cv_maildir=${SHADOW_MAILDIR} 170shadow_cv_mailfile=${SHADOW_MAILFILE} 171shadow_cv_utmpdir=${SHADOW_UTMPDIR} 172shadow_cv_logdir=${SHADOW_LOGDIR} 173shadow_cv_passwd_dir=${bindir} 174END 175""") 176 self.assertReferences(set(v)) 177 self.assertExecs(set(["cat"])) 178 179# def test_incomplete_command_expansion(self): 180# self.assertRaises(reftracker.ShellSyntaxError, reftracker.execs, 181# bbvalue.shparse("cp foo`", self.d), self.d) 182 183# def test_rogue_dollarsign(self): 184# self.setValues({"D" : "/tmp"}) 185# self.parseExpression("install -d ${D}$") 186# self.assertReferences(set(["D"])) 187# self.assertExecs(set(["install"])) 188 189 190class PythonReferenceTest(ReferenceTest): 191 192 def setUp(self): 193 self.d = bb.data.init() 194 if hasattr(bb.utils, "_context"): 195 self.context = bb.utils._context 196 else: 197 import builtins 198 self.context = builtins.__dict__ 199 200 def parseExpression(self, exp): 201 parsedvar = self.d.expandWithRefs(exp, None) 202 parser = bb.codeparser.PythonParser("ParserTest", logger) 203 parser.parse_python(parsedvar.value) 204 205 self.references = parsedvar.references | parser.references 206 self.execs = parser.execs 207 self.contains = parser.contains 208 209 @staticmethod 210 def indent(value): 211 """Python Snippets have to be indented, python values don't have to 212be. These unit tests are testing snippets.""" 213 return " " + value 214 215 def test_getvar_reference(self): 216 self.parseExpression("d.getVar('foo')") 217 self.assertReferences(set(["foo"])) 218 self.assertExecs(set()) 219 220 def test_getvar_computed_reference(self): 221 self.parseExpression("d.getVar('f' + 'o' + 'o')") 222 self.assertReferences(set()) 223 self.assertExecs(set()) 224 225 def test_getvar_exec_reference(self): 226 self.parseExpression("eval('d.getVar(\"foo\")')") 227 self.assertReferences(set()) 228 self.assertExecs(set(["eval"])) 229 230 def test_var_reference(self): 231 self.context["foo"] = lambda x: x 232 self.setEmptyVars(["FOO"]) 233 self.parseExpression("foo('${FOO}')") 234 self.assertReferences(set(["FOO"])) 235 self.assertExecs(set(["foo"])) 236 del self.context["foo"] 237 238 def test_var_exec(self): 239 for etype in ("func", "task"): 240 self.d.setVar("do_something", "echo 'hi mom! ${FOO}'") 241 self.d.setVarFlag("do_something", etype, True) 242 self.parseExpression("bb.build.exec_func('do_something', d)") 243 self.assertReferences(set([])) 244 self.assertExecs(set(["do_something"])) 245 246 def test_function_reference(self): 247 self.context["testfunc"] = lambda msg: bb.msg.note(1, None, msg) 248 self.d.setVar("FOO", "Hello, World!") 249 self.parseExpression("testfunc('${FOO}')") 250 self.assertReferences(set(["FOO"])) 251 self.assertExecs(set(["testfunc"])) 252 del self.context["testfunc"] 253 254 def test_qualified_function_reference(self): 255 self.parseExpression("time.time()") 256 self.assertExecs(set(["time.time"])) 257 258 def test_qualified_function_reference_2(self): 259 self.parseExpression("os.path.dirname('/foo/bar')") 260 self.assertExecs(set(["os.path.dirname"])) 261 262 def test_qualified_function_reference_nested(self): 263 self.parseExpression("time.strftime('%Y%m%d',time.gmtime())") 264 self.assertExecs(set(["time.strftime", "time.gmtime"])) 265 266 def test_function_reference_chained(self): 267 self.context["testget"] = lambda: "\tstrip me " 268 self.parseExpression("testget().strip()") 269 self.assertExecs(set(["testget"])) 270 del self.context["testget"] 271 272 def test_contains(self): 273 self.parseExpression('bb.utils.contains("TESTVAR", "one", "true", "false", d)') 274 self.assertContains({'TESTVAR': {'one'}}) 275 276 def test_contains_multi(self): 277 self.parseExpression('bb.utils.contains("TESTVAR", "one two", "true", "false", d)') 278 self.assertContains({'TESTVAR': {'one two'}}) 279 280 def test_contains_any(self): 281 self.parseExpression('bb.utils.contains_any("TESTVAR", "hello", "true", "false", d)') 282 self.assertContains({'TESTVAR': {'hello'}}) 283 284 def test_contains_any_multi(self): 285 self.parseExpression('bb.utils.contains_any("TESTVAR", "one two three", "true", "false", d)') 286 self.assertContains({'TESTVAR': {'one', 'two', 'three'}}) 287 288 def test_contains_filter(self): 289 self.parseExpression('bb.utils.filter("TESTVAR", "hello there world", d)') 290 self.assertContains({'TESTVAR': {'hello', 'there', 'world'}}) 291 292 293class DependencyReferenceTest(ReferenceTest): 294 295 pydata = """ 296d.getVar('somevar') 297def test(d): 298 foo = 'bar %s' % 'foo' 299def test2(d): 300 d.getVar(foo) 301 d.getVar('bar', False) 302 test2(d) 303 304def a(): 305 \"\"\"some 306 stuff 307 \"\"\" 308 return "heh" 309 310test(d) 311 312d.expand(d.getVar("something", False)) 313d.expand("${inexpand} somethingelse") 314d.getVar(a(), False) 315""" 316 317 def test_python(self): 318 self.d.setVar("FOO", self.pydata) 319 self.setEmptyVars(["inexpand", "a", "test2", "test"]) 320 self.d.setVarFlags("FOO", { 321 "func": True, 322 "python": True, 323 "lineno": 1, 324 "filename": "example.bb", 325 }) 326 327 deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d) 328 329 self.assertEqual(deps, set(["somevar", "bar", "something", "inexpand", "test", "test2", "a"])) 330 331 332 shelldata = """ 333foo () { 334bar 335} 336{ 337echo baz 338$(heh) 339eval `moo` 340} 341a=b 342c=d 343( 344true && false 345test -f foo 346testval=something 347$testval 348) || aiee 349! inverted 350echo ${somevar} 351 352case foo in 353bar) 354echo bar 355;; 356baz) 357echo baz 358;; 359foo*) 360echo foo 361;; 362esac 363""" 364 365 def test_shell(self): 366 execs = ["bar", "echo", "heh", "moo", "true", "aiee"] 367 self.d.setVar("somevar", "heh") 368 self.d.setVar("inverted", "echo inverted...") 369 self.d.setVarFlag("inverted", "func", True) 370 self.d.setVar("FOO", self.shelldata) 371 self.d.setVarFlags("FOO", {"func": True}) 372 self.setEmptyVars(execs) 373 374 deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d) 375 376 self.assertEqual(deps, set(["somevar", "inverted"] + execs)) 377 378 379 def test_vardeps(self): 380 self.d.setVar("oe_libinstall", "echo test") 381 self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") 382 self.d.setVarFlag("FOO", "vardeps", "oe_libinstall") 383 384 deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d) 385 386 self.assertEqual(deps, set(["oe_libinstall"])) 387 388 def test_vardeps_expand(self): 389 self.d.setVar("oe_libinstall", "echo test") 390 self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") 391 self.d.setVarFlag("FOO", "vardeps", "${@'oe_libinstall'}") 392 393 deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d) 394 395 self.assertEqual(deps, set(["oe_libinstall"])) 396 397 def test_contains_vardeps(self): 398 expr = '${@bb.utils.filter("TESTVAR", "somevalue anothervalue", d)} \ 399 ${@bb.utils.contains("TESTVAR", "testval testval2", "yetanothervalue", "", d)} \ 400 ${@bb.utils.contains("TESTVAR", "testval2 testval3", "blah", "", d)} \ 401 ${@bb.utils.contains_any("TESTVAR", "testval2 testval3", "lastone", "", d)}' 402 parsedvar = self.d.expandWithRefs(expr, None) 403 # Check contains 404 self.assertEqual(parsedvar.contains, {'TESTVAR': {'testval2 testval3', 'anothervalue', 'somevalue', 'testval testval2', 'testval2', 'testval3'}}) 405 # Check dependencies 406 self.d.setVar('ANOTHERVAR', expr) 407 self.d.setVar('TESTVAR', 'anothervalue testval testval2') 408 deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d) 409 self.assertEqual(sorted(values.splitlines()), 410 sorted([expr, 411 'TESTVAR{anothervalue} = Set', 412 'TESTVAR{somevalue} = Unset', 413 'TESTVAR{testval testval2} = Set', 414 'TESTVAR{testval2 testval3} = Unset', 415 'TESTVAR{testval2} = Set', 416 'TESTVAR{testval3} = Unset' 417 ])) 418 # Check final value 419 self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['anothervalue', 'yetanothervalue', 'lastone']) 420 421 def test_contains_vardeps_excluded(self): 422 # Check the ignored_vars option to build_dependencies is handled by contains functionality 423 varval = '${TESTVAR2} ${@bb.utils.filter("TESTVAR", "somevalue anothervalue", d)}' 424 self.d.setVar('ANOTHERVAR', varval) 425 self.d.setVar('TESTVAR', 'anothervalue testval testval2') 426 self.d.setVar('TESTVAR2', 'testval3') 427 deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(["TESTVAR"]), self.d, self.d) 428 self.assertEqual(sorted(values.splitlines()), sorted([varval])) 429 self.assertEqual(deps, set(["TESTVAR2"])) 430 self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue']) 431 432 # Check the vardepsexclude flag is handled by contains functionality 433 self.d.setVarFlag('ANOTHERVAR', 'vardepsexclude', 'TESTVAR') 434 deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d) 435 self.assertEqual(sorted(values.splitlines()), sorted([varval])) 436 self.assertEqual(deps, set(["TESTVAR2"])) 437 self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue']) 438 439 def test_contains_vardeps_override_operators(self): 440 # Check override operators handle dependencies correctly with the contains functionality 441 expr_plain = 'testval' 442 expr_prepend = '${@bb.utils.filter("TESTVAR1", "testval1", d)} ' 443 expr_append = ' ${@bb.utils.filter("TESTVAR2", "testval2", d)}' 444 expr_remove = '${@bb.utils.contains("TESTVAR3", "no-testval", "testval", "", d)}' 445 # Check dependencies 446 self.d.setVar('ANOTHERVAR', expr_plain) 447 self.d.prependVar('ANOTHERVAR', expr_prepend) 448 self.d.appendVar('ANOTHERVAR', expr_append) 449 self.d.setVar('ANOTHERVAR:remove', expr_remove) 450 self.d.setVar('TESTVAR1', 'blah') 451 self.d.setVar('TESTVAR2', 'testval2') 452 self.d.setVar('TESTVAR3', 'no-testval') 453 deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d) 454 self.assertEqual(sorted(values.splitlines()), 455 sorted([ 456 expr_prepend + expr_plain + expr_append, 457 '_remove of ' + expr_remove, 458 'TESTVAR1{testval1} = Unset', 459 'TESTVAR2{testval2} = Set', 460 'TESTVAR3{no-testval} = Set', 461 ])) 462 # Check final value 463 self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval2']) 464 465 #Currently no wildcard support 466 #def test_vardeps_wildcards(self): 467 # self.d.setVar("oe_libinstall", "echo test") 468 # self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") 469 # self.d.setVarFlag("FOO", "vardeps", "oe_*") 470 # self.assertEqual(deps, set(["oe_libinstall"])) 471 472 473