1# 2# BitBake Tests for the Fetcher (fetch2/) 3# 4# Copyright (C) 2012 Richard Purdie 5# 6# SPDX-License-Identifier: GPL-2.0-only 7# 8 9import unittest 10import hashlib 11import tempfile 12import collections 13import os 14import signal 15import tarfile 16from bb.fetch2 import URI 17from bb.fetch2 import FetchMethod 18import bb 19from bb.tests.support.httpserver import HTTPService 20 21def skipIfNoNetwork(): 22 if os.environ.get("BB_SKIP_NETTESTS") == "yes": 23 return unittest.skip("network test") 24 return lambda f: f 25 26class TestTimeout(Exception): 27 # Indicate to pytest that this is not a test suite 28 __test__ = False 29 30class Timeout(): 31 32 def __init__(self, seconds): 33 self.seconds = seconds 34 35 def handle_timeout(self, signum, frame): 36 raise TestTimeout("Test failed: timeout reached") 37 38 def __enter__(self): 39 signal.signal(signal.SIGALRM, self.handle_timeout) 40 signal.alarm(self.seconds) 41 42 def __exit__(self, exc_type, exc_val, exc_tb): 43 signal.alarm(0) 44 45class URITest(unittest.TestCase): 46 test_uris = { 47 "http://www.google.com/index.html" : { 48 'uri': 'http://www.google.com/index.html', 49 'scheme': 'http', 50 'hostname': 'www.google.com', 51 'port': None, 52 'hostport': 'www.google.com', 53 'path': '/index.html', 54 'userinfo': '', 55 'username': '', 56 'password': '', 57 'params': {}, 58 'query': {}, 59 'relative': False 60 }, 61 "http://www.google.com/index.html;param1=value1" : { 62 'uri': 'http://www.google.com/index.html;param1=value1', 63 'scheme': 'http', 64 'hostname': 'www.google.com', 65 'port': None, 66 'hostport': 'www.google.com', 67 'path': '/index.html', 68 'userinfo': '', 69 'username': '', 70 'password': '', 71 'params': { 72 'param1': 'value1' 73 }, 74 'query': {}, 75 'relative': False 76 }, 77 "http://www.example.org/index.html?param1=value1" : { 78 'uri': 'http://www.example.org/index.html?param1=value1', 79 'scheme': 'http', 80 'hostname': 'www.example.org', 81 'port': None, 82 'hostport': 'www.example.org', 83 'path': '/index.html', 84 'userinfo': '', 85 'username': '', 86 'password': '', 87 'params': {}, 88 'query': { 89 'param1': 'value1' 90 }, 91 'relative': False 92 }, 93 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2" : { 94 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2', 95 'scheme': 'http', 96 'hostname': 'www.example.org', 97 'port': None, 98 'hostport': 'www.example.org', 99 'path': '/index.html', 100 'userinfo': '', 101 'username': '', 102 'password': '', 103 'params': { 104 'param2': 'value2' 105 }, 106 'query': { 107 'qparam1': 'qvalue1' 108 }, 109 'relative': False 110 }, 111 # Check that trailing semicolons are handled correctly 112 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2;" : { 113 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2', 114 'scheme': 'http', 115 'hostname': 'www.example.org', 116 'port': None, 117 'hostport': 'www.example.org', 118 'path': '/index.html', 119 'userinfo': '', 120 'username': '', 121 'password': '', 122 'params': { 123 'param2': 'value2' 124 }, 125 'query': { 126 'qparam1': 'qvalue1' 127 }, 128 'relative': False 129 }, 130 "http://www.example.com:8080/index.html" : { 131 'uri': 'http://www.example.com:8080/index.html', 132 'scheme': 'http', 133 'hostname': 'www.example.com', 134 'port': 8080, 135 'hostport': 'www.example.com:8080', 136 'path': '/index.html', 137 'userinfo': '', 138 'username': '', 139 'password': '', 140 'params': {}, 141 'query': {}, 142 'relative': False 143 }, 144 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : { 145 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg', 146 'scheme': 'cvs', 147 'hostname': 'cvs.handhelds.org', 148 'port': None, 149 'hostport': 'cvs.handhelds.org', 150 'path': '/cvs', 151 'userinfo': 'anoncvs', 152 'username': 'anoncvs', 153 'password': '', 154 'params': { 155 'module': 'familiar/dist/ipkg' 156 }, 157 'query': {}, 158 'relative': False 159 }, 160 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": { 161 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg', 162 'scheme': 'cvs', 163 'hostname': 'cvs.handhelds.org', 164 'port': None, 165 'hostport': 'cvs.handhelds.org', 166 'path': '/cvs', 167 'userinfo': 'anoncvs:anonymous', 168 'username': 'anoncvs', 169 'password': 'anonymous', 170 'params': collections.OrderedDict([ 171 ('tag', 'V0-99-81'), 172 ('module', 'familiar/dist/ipkg') 173 ]), 174 'query': {}, 175 'relative': False 176 }, 177 "file://example.diff": { # NOTE: Not RFC compliant! 178 'uri': 'file:example.diff', 179 'scheme': 'file', 180 'hostname': '', 181 'port': None, 182 'hostport': '', 183 'path': 'example.diff', 184 'userinfo': '', 185 'username': '', 186 'password': '', 187 'params': {}, 188 'query': {}, 189 'relative': True 190 }, 191 "file:example.diff": { # NOTE: RFC compliant version of the former 192 'uri': 'file:example.diff', 193 'scheme': 'file', 194 'hostname': '', 195 'port': None, 196 'hostport': '', 197 'path': 'example.diff', 198 'userinfo': '', 199 'userinfo': '', 200 'username': '', 201 'password': '', 202 'params': {}, 203 'query': {}, 204 'relative': True 205 }, 206 "file:///tmp/example.diff": { 207 'uri': 'file:///tmp/example.diff', 208 'scheme': 'file', 209 'hostname': '', 210 'port': None, 211 'hostport': '', 212 'path': '/tmp/example.diff', 213 'userinfo': '', 214 'userinfo': '', 215 'username': '', 216 'password': '', 217 'params': {}, 218 'query': {}, 219 'relative': False 220 }, 221 "git:///path/example.git": { 222 'uri': 'git:///path/example.git', 223 'scheme': 'git', 224 'hostname': '', 225 'port': None, 226 'hostport': '', 227 'path': '/path/example.git', 228 'userinfo': '', 229 'userinfo': '', 230 'username': '', 231 'password': '', 232 'params': {}, 233 'query': {}, 234 'relative': False 235 }, 236 "git:path/example.git": { 237 'uri': 'git:path/example.git', 238 'scheme': 'git', 239 'hostname': '', 240 'port': None, 241 'hostport': '', 242 'path': 'path/example.git', 243 'userinfo': '', 244 'userinfo': '', 245 'username': '', 246 'password': '', 247 'params': {}, 248 'query': {}, 249 'relative': True 250 }, 251 "git://example.net/path/example.git": { 252 'uri': 'git://example.net/path/example.git', 253 'scheme': 'git', 254 'hostname': 'example.net', 255 'port': None, 256 'hostport': 'example.net', 257 'path': '/path/example.git', 258 'userinfo': '', 259 'userinfo': '', 260 'username': '', 261 'password': '', 262 'params': {}, 263 'query': {}, 264 'relative': False 265 }, 266 "git://tfs-example.org:22/tfs/example%20path/example.git": { 267 'uri': 'git://tfs-example.org:22/tfs/example%20path/example.git', 268 'scheme': 'git', 269 'hostname': 'tfs-example.org', 270 'port': 22, 271 'hostport': 'tfs-example.org:22', 272 'path': '/tfs/example path/example.git', 273 'userinfo': '', 274 'userinfo': '', 275 'username': '', 276 'password': '', 277 'params': {}, 278 'query': {}, 279 'relative': False 280 }, 281 "http://somesite.net;someparam=1": { 282 'uri': 'http://somesite.net;someparam=1', 283 'scheme': 'http', 284 'hostname': 'somesite.net', 285 'port': None, 286 'hostport': 'somesite.net', 287 'path': '', 288 'userinfo': '', 289 'userinfo': '', 290 'username': '', 291 'password': '', 292 'params': {"someparam" : "1"}, 293 'query': {}, 294 'relative': False 295 }, 296 "file://somelocation;someparam=1": { 297 'uri': 'file:somelocation;someparam=1', 298 'scheme': 'file', 299 'hostname': '', 300 'port': None, 301 'hostport': '', 302 'path': 'somelocation', 303 'userinfo': '', 304 'userinfo': '', 305 'username': '', 306 'password': '', 307 'params': {"someparam" : "1"}, 308 'query': {}, 309 'relative': True 310 } 311 312 } 313 314 def test_uri(self): 315 for test_uri, ref in self.test_uris.items(): 316 uri = URI(test_uri) 317 318 self.assertEqual(str(uri), ref['uri']) 319 320 # expected attributes 321 self.assertEqual(uri.scheme, ref['scheme']) 322 323 self.assertEqual(uri.userinfo, ref['userinfo']) 324 self.assertEqual(uri.username, ref['username']) 325 self.assertEqual(uri.password, ref['password']) 326 327 self.assertEqual(uri.hostname, ref['hostname']) 328 self.assertEqual(uri.port, ref['port']) 329 self.assertEqual(uri.hostport, ref['hostport']) 330 331 self.assertEqual(uri.path, ref['path']) 332 self.assertEqual(uri.params, ref['params']) 333 334 self.assertEqual(uri.relative, ref['relative']) 335 336 def test_dict(self): 337 for test in self.test_uris.values(): 338 uri = URI() 339 340 self.assertEqual(uri.scheme, '') 341 self.assertEqual(uri.userinfo, '') 342 self.assertEqual(uri.username, '') 343 self.assertEqual(uri.password, '') 344 self.assertEqual(uri.hostname, '') 345 self.assertEqual(uri.port, None) 346 self.assertEqual(uri.path, '') 347 self.assertEqual(uri.params, {}) 348 349 350 uri.scheme = test['scheme'] 351 self.assertEqual(uri.scheme, test['scheme']) 352 353 uri.userinfo = test['userinfo'] 354 self.assertEqual(uri.userinfo, test['userinfo']) 355 self.assertEqual(uri.username, test['username']) 356 self.assertEqual(uri.password, test['password']) 357 358 # make sure changing the values doesn't do anything unexpected 359 uri.username = 'changeme' 360 self.assertEqual(uri.username, 'changeme') 361 self.assertEqual(uri.password, test['password']) 362 uri.password = 'insecure' 363 self.assertEqual(uri.username, 'changeme') 364 self.assertEqual(uri.password, 'insecure') 365 366 # reset back after our trickery 367 uri.userinfo = test['userinfo'] 368 self.assertEqual(uri.userinfo, test['userinfo']) 369 self.assertEqual(uri.username, test['username']) 370 self.assertEqual(uri.password, test['password']) 371 372 uri.hostname = test['hostname'] 373 self.assertEqual(uri.hostname, test['hostname']) 374 self.assertEqual(uri.hostport, test['hostname']) 375 376 uri.port = test['port'] 377 self.assertEqual(uri.port, test['port']) 378 self.assertEqual(uri.hostport, test['hostport']) 379 380 uri.path = test['path'] 381 self.assertEqual(uri.path, test['path']) 382 383 uri.params = test['params'] 384 self.assertEqual(uri.params, test['params']) 385 386 uri.query = test['query'] 387 self.assertEqual(uri.query, test['query']) 388 389 self.assertEqual(str(uri), test['uri']) 390 391 uri.params = {} 392 self.assertEqual(uri.params, {}) 393 self.assertEqual(str(uri), (str(uri).split(";"))[0]) 394 395class FetcherTest(unittest.TestCase): 396 397 def setUp(self): 398 self.origdir = os.getcwd() 399 self.d = bb.data.init() 400 self.tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-") 401 self.dldir = os.path.join(self.tempdir, "download") 402 os.mkdir(self.dldir) 403 self.d.setVar("DL_DIR", self.dldir) 404 self.unpackdir = os.path.join(self.tempdir, "unpacked") 405 os.mkdir(self.unpackdir) 406 persistdir = os.path.join(self.tempdir, "persistdata") 407 self.d.setVar("PERSISTENT_DIR", persistdir) 408 409 def tearDown(self): 410 os.chdir(self.origdir) 411 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes": 412 print("Not cleaning up %s. Please remove manually." % self.tempdir) 413 else: 414 bb.process.run('chmod u+rw -R %s' % self.tempdir) 415 bb.utils.prunedir(self.tempdir) 416 417 def git(self, cmd, cwd=None): 418 if isinstance(cmd, str): 419 cmd = 'git -c safe.bareRepository=all ' + cmd 420 else: 421 cmd = ['git', '-c', 'safe.bareRepository=all'] + cmd 422 if cwd is None: 423 cwd = self.gitdir 424 return bb.process.run(cmd, cwd=cwd)[0] 425 426 def git_init(self, cwd=None): 427 self.git('init', cwd=cwd) 428 # Explicitly set initial branch to master as 429 # a common setup is to use other default 430 # branch than master. 431 self.git(['checkout', '-b', 'master'], cwd=cwd) 432 433 try: 434 self.git(['config', 'user.email'], cwd=cwd) 435 except bb.process.ExecutionError: 436 self.git(['config', 'user.email', 'you@example.com'], cwd=cwd) 437 438 try: 439 self.git(['config', 'user.name'], cwd=cwd) 440 except bb.process.ExecutionError: 441 self.git(['config', 'user.name', 'Your Name'], cwd=cwd) 442 443class MirrorUriTest(FetcherTest): 444 445 replaceuris = { 446 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/") 447 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz", 448 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 449 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 450 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 451 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 452 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http") 453 : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 454 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake") 455 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890", 456 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache") 457 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 458 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/") 459 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 460 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3") 461 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", 462 ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz") 463 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", 464 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist") 465 : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", 466 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/") 467 : "file:///somepath/downloads/subversion-1.7.1.tar.bz2", 468 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 469 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 470 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 471 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 472 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http") 473 : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 474 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org") 475 : "http://somewhere2.org/somefile_1.2.3.tar.gz", 476 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/") 477 : "http://somewhere2.org/somefile_1.2.3.tar.gz", 478 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http") 479 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 480 ("git://user1@someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://user2@git.openembedded.org/bitbake;protocol=http") 481 : "git://user2@git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 482 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=git;branch=master", "git://someserver.org/bitbake", "git://someotherserver.org/bitbake;protocol=https") 483 : "git://someotherserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=https;branch=master", 484 ("gitsm://git.qemu.org/git/seabios.git/;protocol=https;name=roms/seabios;subpath=roms/seabios;bareclone=1;nobranch=1;rev=1234567890123456789012345678901234567890", "gitsm://.*/.*", "http://petalinux.xilinx.com/sswreleases/rel-v${XILINX_VER_MAIN}/downloads") : "http://petalinux.xilinx.com/sswreleases/rel-v%24%7BXILINX_VER_MAIN%7D/downloads/git2_git.qemu.org.git.seabios.git..tar.gz", 485 ("https://somewhere.org/example/1.0.0/example;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/PATH") 486 : "file:///mirror/example/1.0.0/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", 487 ("https://somewhere.org/example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/some-example-1.0.0.tgz") 488 : "file:///mirror/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", 489 490 #Renaming files doesn't work 491 #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz" 492 #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 493 } 494 495 mirrorvar = "http://.*/.* file:///somepath/downloads/ " \ 496 "git://someserver.org/bitbake git://git.openembedded.org/bitbake " \ 497 "https://.*/.* file:///someotherpath/downloads/ " \ 498 "http://.*/.* file:///someotherpath/downloads/" 499 500 def test_urireplace(self): 501 self.d.setVar("FILESPATH", ".") 502 for k, v in self.replaceuris.items(): 503 ud = bb.fetch.FetchData(k[0], self.d) 504 ud.setup_localpath(self.d) 505 mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2])) 506 newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d) 507 self.assertEqual([v], newuris) 508 509 def test_urilist1(self): 510 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 511 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) 512 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 513 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz']) 514 515 def test_urilist2(self): 516 # Catch https:// -> files:// bug 517 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 518 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) 519 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 520 self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz']) 521 522 def test_mirror_of_mirror(self): 523 # Test if mirror of a mirror works 524 mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/" 525 mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/" 526 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 527 mirrors = bb.fetch2.mirror_from_string(mirrorvar) 528 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 529 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 530 'file:///someotherpath/downloads/bitbake-1.0.tar.gz', 531 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz', 532 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz']) 533 534 recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ " \ 535 "https://.*/[^/]* https://BBBB/B/B/B/" 536 537 def test_recursive(self): 538 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 539 mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar) 540 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 541 self.assertEqual(uris, ['http://AAAA/A/A/A/bitbake/bitbake-1.0.tar.gz', 542 'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz', 543 'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz']) 544 545 546class GitDownloadDirectoryNamingTest(FetcherTest): 547 def setUp(self): 548 super(GitDownloadDirectoryNamingTest, self).setUp() 549 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 550 self.recipe_dir = "git.openembedded.org.bitbake" 551 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 552 self.mirror_dir = "github.com.openembedded.bitbake.git" 553 554 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 555 556 def setup_mirror_rewrite(self): 557 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 558 559 @skipIfNoNetwork() 560 def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self): 561 self.setup_mirror_rewrite() 562 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 563 564 fetcher.download() 565 566 dir = os.listdir(self.dldir + "/git2") 567 self.assertIn(self.recipe_dir, dir) 568 569 @skipIfNoNetwork() 570 def test_that_directory_exists_for_mirrored_url_and_recipe_url_when_mirroring_is_used(self): 571 self.setup_mirror_rewrite() 572 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 573 574 fetcher.download() 575 576 dir = os.listdir(self.dldir + "/git2") 577 self.assertIn(self.mirror_dir, dir) 578 self.assertIn(self.recipe_dir, dir) 579 580 @skipIfNoNetwork() 581 def test_that_recipe_directory_and_mirrored_directory_exists_when_mirroring_is_used_and_the_mirrored_directory_already_exists(self): 582 self.setup_mirror_rewrite() 583 fetcher = bb.fetch.Fetch([self.mirror_url], self.d) 584 fetcher.download() 585 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 586 587 fetcher.download() 588 589 dir = os.listdir(self.dldir + "/git2") 590 self.assertIn(self.mirror_dir, dir) 591 self.assertIn(self.recipe_dir, dir) 592 593 594class TarballNamingTest(FetcherTest): 595 def setUp(self): 596 super(TarballNamingTest, self).setUp() 597 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 598 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz" 599 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 600 self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz" 601 602 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1') 603 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 604 605 def setup_mirror_rewrite(self): 606 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 607 608 @skipIfNoNetwork() 609 def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self): 610 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 611 612 fetcher.download() 613 614 dir = os.listdir(self.dldir) 615 self.assertIn(self.recipe_tarball, dir) 616 617 @skipIfNoNetwork() 618 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self): 619 self.setup_mirror_rewrite() 620 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 621 622 fetcher.download() 623 624 dir = os.listdir(self.dldir) 625 self.assertIn(self.mirror_tarball, dir) 626 627 628class GitShallowTarballNamingTest(FetcherTest): 629 def setUp(self): 630 super(GitShallowTarballNamingTest, self).setUp() 631 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 632 self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz" 633 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 634 self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz" 635 636 self.d.setVar('BB_GIT_SHALLOW', '1') 637 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 638 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 639 640 def setup_mirror_rewrite(self): 641 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 642 643 @skipIfNoNetwork() 644 def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self): 645 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 646 647 fetcher.download() 648 649 dir = os.listdir(self.dldir) 650 self.assertIn(self.recipe_tarball, dir) 651 652 @skipIfNoNetwork() 653 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self): 654 self.setup_mirror_rewrite() 655 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 656 657 fetcher.download() 658 659 dir = os.listdir(self.dldir) 660 self.assertIn(self.mirror_tarball, dir) 661 662 663class CleanTarballTest(FetcherTest): 664 def setUp(self): 665 super(CleanTarballTest, self).setUp() 666 self.recipe_url = "git://git.openembedded.org/bitbake;protocol=https" 667 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz" 668 669 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1') 670 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 671 672 @skipIfNoNetwork() 673 def test_that_the_tarball_contents_does_not_leak_info(self): 674 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 675 676 fetcher.download() 677 678 fetcher.unpack(self.unpackdir) 679 mtime = bb.process.run('git log --all -1 --format=%ct', 680 cwd=os.path.join(self.unpackdir, 'git')) 681 self.assertEqual(len(mtime), 2) 682 mtime = int(mtime[0]) 683 684 archive = tarfile.open(os.path.join(self.dldir, self.recipe_tarball)) 685 self.assertNotEqual(len(archive.members), 0) 686 for member in archive.members: 687 if member.name == ".": 688 continue 689 self.assertEqual(member.uname, 'oe', "user name for %s differs" % member.name) 690 self.assertEqual(member.uid, 0, "uid for %s differs" % member.name) 691 self.assertEqual(member.gname, 'oe', "group name for %s differs" % member.name) 692 self.assertEqual(member.gid, 0, "gid for %s differs" % member.name) 693 self.assertEqual(member.mtime, mtime, "mtime for %s differs" % member.name) 694 695 696class FetcherLocalTest(FetcherTest): 697 def setUp(self): 698 def touch(fn): 699 with open(fn, 'a'): 700 os.utime(fn, None) 701 702 super(FetcherLocalTest, self).setUp() 703 self.localsrcdir = os.path.join(self.tempdir, 'localsrc') 704 os.makedirs(self.localsrcdir) 705 touch(os.path.join(self.localsrcdir, 'a')) 706 touch(os.path.join(self.localsrcdir, 'b')) 707 os.makedirs(os.path.join(self.localsrcdir, 'dir')) 708 touch(os.path.join(self.localsrcdir, 'dir', 'c')) 709 touch(os.path.join(self.localsrcdir, 'dir', 'd')) 710 os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir')) 711 touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e')) 712 touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device')) 713 bb.process.run('tar cf archive.tar -C dir .', cwd=self.localsrcdir) 714 bb.process.run('tar czf archive.tar.gz -C dir .', cwd=self.localsrcdir) 715 bb.process.run('tar cjf archive.tar.bz2 -C dir .', cwd=self.localsrcdir) 716 self.d.setVar("FILESPATH", self.localsrcdir) 717 718 def fetchUnpack(self, uris): 719 fetcher = bb.fetch.Fetch(uris, self.d) 720 fetcher.download() 721 fetcher.unpack(self.unpackdir) 722 flst = [] 723 for root, dirs, files in os.walk(self.unpackdir): 724 for f in files: 725 flst.append(os.path.relpath(os.path.join(root, f), self.unpackdir)) 726 flst.sort() 727 return flst 728 729 def test_local_checksum_fails_no_file(self): 730 self.d.setVar("SRC_URI", "file://404") 731 with self.assertRaises(bb.BBHandledException): 732 bb.fetch.get_checksum_file_list(self.d) 733 734 def test_local(self): 735 tree = self.fetchUnpack(['file://a', 'file://dir/c']) 736 self.assertEqual(tree, ['a', 'dir/c']) 737 738 def test_local_backslash(self): 739 tree = self.fetchUnpack([r'file://backslash\x2dsystemd-unit.device']) 740 self.assertEqual(tree, [r'backslash\x2dsystemd-unit.device']) 741 742 def test_local_wildcard(self): 743 with self.assertRaises(bb.fetch2.ParameterError): 744 tree = self.fetchUnpack(['file://a', 'file://dir/*']) 745 746 def test_local_dir(self): 747 tree = self.fetchUnpack(['file://a', 'file://dir']) 748 self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e']) 749 750 def test_local_subdir(self): 751 tree = self.fetchUnpack(['file://dir/subdir']) 752 self.assertEqual(tree, ['dir/subdir/e']) 753 754 def test_local_subdir_file(self): 755 tree = self.fetchUnpack(['file://dir/subdir/e']) 756 self.assertEqual(tree, ['dir/subdir/e']) 757 758 def test_local_subdirparam(self): 759 tree = self.fetchUnpack(['file://a;subdir=bar', 'file://dir;subdir=foo/moo']) 760 self.assertEqual(tree, ['bar/a', 'foo/moo/dir/c', 'foo/moo/dir/d', 'foo/moo/dir/subdir/e']) 761 762 def test_local_deepsubdirparam(self): 763 tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar']) 764 self.assertEqual(tree, ['bar/dir/subdir/e']) 765 766 def test_local_absolutedir(self): 767 # Unpacking to an absolute path that is a subdirectory of the root 768 # should work 769 tree = self.fetchUnpack(['file://a;subdir=%s' % os.path.join(self.unpackdir, 'bar')]) 770 771 # Unpacking to an absolute path outside of the root should fail 772 with self.assertRaises(bb.fetch2.UnpackError): 773 self.fetchUnpack(['file://a;subdir=/bin/sh']) 774 775 def test_local_striplevel(self): 776 tree = self.fetchUnpack(['file://archive.tar;subdir=bar;striplevel=1']) 777 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 778 779 def test_local_striplevel_gzip(self): 780 tree = self.fetchUnpack(['file://archive.tar.gz;subdir=bar;striplevel=1']) 781 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 782 783 def test_local_striplevel_bzip2(self): 784 tree = self.fetchUnpack(['file://archive.tar.bz2;subdir=bar;striplevel=1']) 785 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 786 787 def dummyGitTest(self, suffix): 788 # Create dummy local Git repo 789 src_dir = tempfile.mkdtemp(dir=self.tempdir, 790 prefix='gitfetch_localusehead_') 791 self.gitdir = os.path.abspath(src_dir) 792 self.git_init() 793 self.git(['commit', '--allow-empty', '-m', 'Dummy commit']) 794 # Use other branch than master 795 self.git(['checkout', '-b', 'my-devel']) 796 self.git(['commit', '--allow-empty', '-m', 'Dummy commit 2']) 797 orig_rev = self.git(['rev-parse', 'HEAD']).strip() 798 799 # Fetch and check revision 800 self.d.setVar("SRCREV", "AUTOINC") 801 self.d.setVar("__BBSRCREV_SEEN", "1") 802 url = "git://" + self.gitdir + ";branch=master;protocol=file;" + suffix 803 fetcher = bb.fetch.Fetch([url], self.d) 804 fetcher.download() 805 fetcher.unpack(self.unpackdir) 806 unpack_rev = self.git(['rev-parse', 'HEAD'], 807 cwd=os.path.join(self.unpackdir, 'git')).strip() 808 self.assertEqual(orig_rev, unpack_rev) 809 810 def test_local_gitfetch_usehead(self): 811 self.dummyGitTest("usehead=1") 812 813 def test_local_gitfetch_usehead_withname(self): 814 self.dummyGitTest("usehead=1;name=newName") 815 816 def test_local_gitfetch_shared(self): 817 self.dummyGitTest("usehead=1;name=sharedName") 818 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 819 self.assertTrue(os.path.exists(alt)) 820 821 def test_local_gitfetch_noshared(self): 822 self.d.setVar('BB_GIT_NOSHARED', '1') 823 self.unpackdir += '_noshared' 824 self.dummyGitTest("usehead=1;name=noSharedName") 825 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 826 self.assertFalse(os.path.exists(alt)) 827 828class FetcherNoNetworkTest(FetcherTest): 829 def setUp(self): 830 super().setUp() 831 # all test cases are based on not having network 832 self.d.setVar("BB_NO_NETWORK", "1") 833 834 def test_missing(self): 835 string = "this is a test file\n".encode("utf-8") 836 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 837 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 838 839 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 840 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 841 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 842 with self.assertRaises(bb.fetch2.NetworkAccess): 843 fetcher.download() 844 845 def test_valid_missing_donestamp(self): 846 # create the file in the download directory with correct hash 847 string = "this is a test file\n".encode("utf-8") 848 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb") as f: 849 f.write(string) 850 851 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 852 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 853 854 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 855 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 856 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 857 fetcher.download() 858 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 859 860 def test_invalid_missing_donestamp(self): 861 # create an invalid file in the download directory with incorrect hash 862 string = "this is a test file\n".encode("utf-8") 863 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 864 pass 865 866 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 867 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 868 869 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 870 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 871 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 872 with self.assertRaises(bb.fetch2.NetworkAccess): 873 fetcher.download() 874 # the existing file should not exist or should have be moved to "bad-checksum" 875 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 876 877 def test_nochecksums_missing(self): 878 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 879 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 880 # ssh fetch does not support checksums 881 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 882 # attempts to download with missing donestamp 883 with self.assertRaises(bb.fetch2.NetworkAccess): 884 fetcher.download() 885 886 def test_nochecksums_missing_donestamp(self): 887 # create a file in the download directory 888 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 889 pass 890 891 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 892 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 893 # ssh fetch does not support checksums 894 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 895 # attempts to download with missing donestamp 896 with self.assertRaises(bb.fetch2.NetworkAccess): 897 fetcher.download() 898 899 def test_nochecksums_has_donestamp(self): 900 # create a file in the download directory with the donestamp 901 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 902 pass 903 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"): 904 pass 905 906 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 907 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 908 # ssh fetch does not support checksums 909 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 910 # should not fetch 911 fetcher.download() 912 # both files should still exist 913 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 914 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 915 916 def test_nochecksums_missing_has_donestamp(self): 917 # create a file in the download directory with the donestamp 918 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"): 919 pass 920 921 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 922 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 923 # ssh fetch does not support checksums 924 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 925 with self.assertRaises(bb.fetch2.NetworkAccess): 926 fetcher.download() 927 # both files should still exist 928 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 929 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 930 931class FetcherNetworkTest(FetcherTest): 932 @skipIfNoNetwork() 933 def test_fetch(self): 934 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) 935 fetcher.download() 936 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 937 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892) 938 self.d.setVar("BB_NO_NETWORK", "1") 939 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) 940 fetcher.download() 941 fetcher.unpack(self.unpackdir) 942 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9) 943 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9) 944 945 @skipIfNoNetwork() 946 def test_fetch_mirror(self): 947 self.d.setVar("MIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 948 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 949 fetcher.download() 950 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 951 952 @skipIfNoNetwork() 953 def test_fetch_mirror_of_mirror(self): 954 self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ http://invalid2.yoctoproject.org/.* https://downloads.yoctoproject.org/releases/bitbake") 955 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 956 fetcher.download() 957 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 958 959 @skipIfNoNetwork() 960 def test_fetch_file_mirror_of_mirror(self): 961 self.d.setVar("FILESPATH", ".") 962 self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ file:///some1where/.* file://some2where/ file://some2where/.* https://downloads.yoctoproject.org/releases/bitbake") 963 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 964 os.mkdir(self.dldir + "/some2where") 965 fetcher.download() 966 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 967 968 @skipIfNoNetwork() 969 def test_fetch_premirror(self): 970 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 971 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 972 fetcher.download() 973 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 974 975 @skipIfNoNetwork() 976 def test_fetch_specify_downloadfilename(self): 977 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d) 978 fetcher.download() 979 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749) 980 981 @skipIfNoNetwork() 982 def test_fetch_premirror_specify_downloadfilename_regex_uri(self): 983 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake/") 984 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 985 fetcher.download() 986 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 987 988 @skipIfNoNetwork() 989 # BZ13039 990 def test_fetch_premirror_specify_downloadfilename_specific_uri(self): 991 self.d.setVar("PREMIRRORS", "http://invalid.yoctoproject.org/releases/bitbake https://downloads.yoctoproject.org/releases/bitbake") 992 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 993 fetcher.download() 994 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 995 996 @skipIfNoNetwork() 997 def test_fetch_premirror_use_downloadfilename_to_fetch(self): 998 # Ensure downloadfilename is used when fetching from premirror. 999 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 1000 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 1001 fetcher.download() 1002 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1003 1004 @skipIfNoNetwork() 1005 def gitfetcher(self, url1, url2): 1006 def checkrevision(self, fetcher): 1007 fetcher.unpack(self.unpackdir) 1008 revision = self.git(['rev-parse', 'HEAD'], 1009 cwd=os.path.join(self.unpackdir, 'git')).strip() 1010 self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5") 1011 1012 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") 1013 self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5") 1014 fetcher = bb.fetch.Fetch([url1], self.d) 1015 fetcher.download() 1016 checkrevision(self, fetcher) 1017 # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works 1018 bb.utils.prunedir(self.dldir + "/git2/") 1019 bb.utils.prunedir(self.unpackdir) 1020 self.d.setVar("BB_NO_NETWORK", "1") 1021 fetcher = bb.fetch.Fetch([url2], self.d) 1022 fetcher.download() 1023 checkrevision(self, fetcher) 1024 1025 @skipIfNoNetwork() 1026 def test_gitfetch(self): 1027 url1 = url2 = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 1028 self.gitfetcher(url1, url2) 1029 1030 @skipIfNoNetwork() 1031 def test_gitfetch_goodsrcrev(self): 1032 # SRCREV is set but matches rev= parameter 1033 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https" 1034 self.gitfetcher(url1, url2) 1035 1036 @skipIfNoNetwork() 1037 def test_gitfetch_badsrcrev(self): 1038 # SRCREV is set but does not match rev= parameter 1039 url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https" 1040 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) 1041 1042 @skipIfNoNetwork() 1043 def test_gitfetch_tagandrev(self): 1044 # SRCREV is set but does not match rev= parameter 1045 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5;protocol=https" 1046 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) 1047 1048 @skipIfNoNetwork() 1049 def test_gitfetch_usehead(self): 1050 # Since self.gitfetcher() sets SRCREV we expect this to override 1051 # `usehead=1' and instead fetch the specified SRCREV. See 1052 # test_local_gitfetch_usehead() for a positive use of the usehead 1053 # feature. 1054 url = "git://git.openembedded.org/bitbake;usehead=1;branch=master;protocol=https" 1055 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url) 1056 1057 @skipIfNoNetwork() 1058 def test_gitfetch_usehead_withname(self): 1059 # Since self.gitfetcher() sets SRCREV we expect this to override 1060 # `usehead=1' and instead fetch the specified SRCREV. See 1061 # test_local_gitfetch_usehead() for a positive use of the usehead 1062 # feature. 1063 url = "git://git.openembedded.org/bitbake;usehead=1;name=newName;branch=master;protocol=https" 1064 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url) 1065 1066 @skipIfNoNetwork() 1067 def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self): 1068 recipeurl = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 1069 mirrorurl = "git://someserver.org/bitbake;branch=master;protocol=https" 1070 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake") 1071 self.gitfetcher(recipeurl, mirrorurl) 1072 1073 @skipIfNoNetwork() 1074 def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self): 1075 recipeurl = "git://someserver.org/bitbake;branch=master;protocol=https" 1076 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake") 1077 self.gitfetcher(recipeurl, recipeurl) 1078 1079 @skipIfNoNetwork() 1080 def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self): 1081 realurl = "https://git.openembedded.org/bitbake" 1082 recipeurl = "git://someserver.org/bitbake;protocol=https" 1083 self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git") 1084 os.chdir(self.tempdir) 1085 self.git(['clone', realurl, self.sourcedir], cwd=self.tempdir) 1086 self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file" % (recipeurl, self.sourcedir)) 1087 self.gitfetcher(recipeurl, recipeurl) 1088 1089 @skipIfNoNetwork() 1090 def test_git_submodule(self): 1091 # URL with ssh submodules 1092 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7;branch=master;protocol=https" 1093 # Original URL (comment this if you have ssh access to git.yoctoproject.org) 1094 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee;branch=master;protocol=https" 1095 fetcher = bb.fetch.Fetch([url], self.d) 1096 fetcher.download() 1097 # Previous cwd has been deleted 1098 os.chdir(os.path.dirname(self.unpackdir)) 1099 fetcher.unpack(self.unpackdir) 1100 1101 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1102 self.assertTrue(os.path.exists(repo_path), msg='Unpacked repository missing') 1103 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake')), msg='bitbake submodule missing') 1104 self.assertFalse(os.path.exists(os.path.join(repo_path, 'na')), msg='uninitialized submodule present') 1105 1106 # Only when we're running the extended test with a submodule's submodule, can we check this. 1107 if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')): 1108 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing') 1109 1110 @skipIfNoNetwork() 1111 def test_git_submodule_restricted_network_premirrors(self): 1112 # this test is to ensure that premirrors will be tried in restricted network 1113 # that is, BB_ALLOWED_NETWORKS does not contain the domain the url uses 1114 url = "gitsm://github.com/grpc/grpc.git;protocol=https;name=grpc;branch=v1.60.x;rev=0ef13a7555dbaadd4633399242524129eef5e231" 1115 # create a download directory to be used as premirror later 1116 tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-") 1117 dl_premirror = os.path.join(tempdir, "download-premirror") 1118 os.mkdir(dl_premirror) 1119 self.d.setVar("DL_DIR", dl_premirror) 1120 fetcher = bb.fetch.Fetch([url], self.d) 1121 fetcher.download() 1122 # now use the premirror in restricted network 1123 self.d.setVar("DL_DIR", self.dldir) 1124 self.d.setVar("PREMIRRORS", "gitsm://.*/.* gitsm://%s/git2/MIRRORNAME;protocol=file" % dl_premirror) 1125 self.d.setVar("BB_ALLOWED_NETWORKS", "*.some.domain") 1126 fetcher = bb.fetch.Fetch([url], self.d) 1127 fetcher.download() 1128 1129 @skipIfNoNetwork() 1130 def test_git_submodule_dbus_broker(self): 1131 # The following external repositories have show failures in fetch and unpack operations 1132 # We want to avoid regressions! 1133 url = "gitsm://github.com/bus1/dbus-broker;protocol=https;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main" 1134 fetcher = bb.fetch.Fetch([url], self.d) 1135 fetcher.download() 1136 # Previous cwd has been deleted 1137 os.chdir(os.path.dirname(self.unpackdir)) 1138 fetcher.unpack(self.unpackdir) 1139 1140 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1141 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-dvar/config')), msg='Missing submodule config "subprojects/c-dvar"') 1142 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-list/config')), msg='Missing submodule config "subprojects/c-list"') 1143 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-rbtree/config')), msg='Missing submodule config "subprojects/c-rbtree"') 1144 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"') 1145 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"') 1146 1147 @skipIfNoNetwork() 1148 def test_git_submodule_CLI11(self): 1149 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main" 1150 fetcher = bb.fetch.Fetch([url], self.d) 1151 fetcher.download() 1152 # Previous cwd has been deleted 1153 os.chdir(os.path.dirname(self.unpackdir)) 1154 fetcher.unpack(self.unpackdir) 1155 1156 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1157 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"') 1158 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') 1159 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') 1160 1161 @skipIfNoNetwork() 1162 def test_git_submodule_update_CLI11(self): 1163 """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """ 1164 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main" 1165 fetcher = bb.fetch.Fetch([url], self.d) 1166 fetcher.download() 1167 1168 # CLI11 that pulls in a newer nlohmann-json 1169 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main" 1170 fetcher = bb.fetch.Fetch([url], self.d) 1171 fetcher.download() 1172 # Previous cwd has been deleted 1173 os.chdir(os.path.dirname(self.unpackdir)) 1174 fetcher.unpack(self.unpackdir) 1175 1176 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1177 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"') 1178 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') 1179 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') 1180 1181 @skipIfNoNetwork() 1182 def test_git_submodule_aktualizr(self): 1183 url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=https;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44" 1184 fetcher = bb.fetch.Fetch([url], self.d) 1185 fetcher.download() 1186 # Previous cwd has been deleted 1187 os.chdir(os.path.dirname(self.unpackdir)) 1188 fetcher.unpack(self.unpackdir) 1189 1190 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1191 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule config "partial/extern/isotp-c/config"') 1192 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')), msg='Missing submodule config "partial/extern/isotp-c/modules/deps/bitfield-c/config"') 1193 self.assertTrue(os.path.exists(os.path.join(repo_path, 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of submodule isotp-c did not unpack properly") 1194 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule config "tests/tuf-test-vectors/config"') 1195 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"') 1196 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"') 1197 1198 @skipIfNoNetwork() 1199 def test_git_submodule_iotedge(self): 1200 """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """ 1201 1202 # This repository also has submodules where the module (name), path and url do not align 1203 url = "gitsm://github.com/azure/iotedge.git;protocol=https;rev=d76e0316c6f324345d77c48a83ce836d09392699;branch=main" 1204 fetcher = bb.fetch.Fetch([url], self.d) 1205 fetcher.download() 1206 # Previous cwd has been deleted 1207 os.chdir(os.path.dirname(self.unpackdir)) 1208 fetcher.unpack(self.unpackdir) 1209 1210 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1211 1212 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/README.md')), msg='Missing submodule checkout') 1213 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/ctest/README.md')), msg='Missing submodule checkout') 1214 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/testrunner/readme.md')), msg='Missing submodule checkout') 1215 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/readme.md')), msg='Missing submodule checkout') 1216 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') 1217 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') 1218 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/README.md')), msg='Missing submodule checkout') 1219 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/README.md')), msg='Missing submodule checkout') 1220 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/ctest/README.md')), msg='Missing submodule checkout') 1221 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/testrunner/readme.md')), msg='Missing submodule checkout') 1222 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/readme.md')), msg='Missing submodule checkout') 1223 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') 1224 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') 1225 1226 @skipIfNoNetwork() 1227 def test_git_submodule_reference_to_parent(self): 1228 self.recipe_url = "gitsm://github.com/gflags/gflags.git;protocol=https;branch=master" 1229 self.d.setVar("SRCREV", "14e1138441bbbb584160cb1c0a0426ec1bac35f1") 1230 with Timeout(60): 1231 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 1232 with self.assertRaises(bb.fetch2.FetchError): 1233 fetcher.download() 1234 1235class SVNTest(FetcherTest): 1236 def skipIfNoSvn(): 1237 import shutil 1238 if not shutil.which("svn"): 1239 return unittest.skip("svn not installed, tests being skipped") 1240 1241 if not shutil.which("svnadmin"): 1242 return unittest.skip("svnadmin not installed, tests being skipped") 1243 1244 return lambda f: f 1245 1246 @skipIfNoSvn() 1247 def setUp(self): 1248 """ Create a local repository """ 1249 1250 super(SVNTest, self).setUp() 1251 1252 # Create something we can fetch 1253 src_dir = tempfile.mkdtemp(dir=self.tempdir, 1254 prefix='svnfetch_srcdir_') 1255 src_dir = os.path.abspath(src_dir) 1256 bb.process.run("echo readme > README.md", cwd=src_dir) 1257 1258 # Store it in a local SVN repository 1259 repo_dir = tempfile.mkdtemp(dir=self.tempdir, 1260 prefix='svnfetch_localrepo_') 1261 repo_dir = os.path.abspath(repo_dir) 1262 bb.process.run("svnadmin create project", cwd=repo_dir) 1263 1264 self.repo_url = "file://%s/project" % repo_dir 1265 bb.process.run("svn import --non-interactive -m 'Initial import' %s %s/trunk" % (src_dir, self.repo_url), 1266 cwd=repo_dir) 1267 1268 bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir) 1269 # Github won't emulate SVN anymore (see https://github.blog/2023-01-20-sunsetting-subversion-support/) 1270 # Use still accessible svn repo (only trunk to avoid longer downloads) 1271 bb.process.run("svn propset svn:externals 'bitbake https://svn.apache.org/repos/asf/serf/trunk' .", 1272 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) 1273 bb.process.run("svn commit --non-interactive -m 'Add external'", 1274 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) 1275 1276 self.src_dir = src_dir 1277 self.repo_dir = repo_dir 1278 1279 @skipIfNoSvn() 1280 def tearDown(self): 1281 os.chdir(self.origdir) 1282 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes": 1283 print("Not cleaning up %s. Please remove manually." % self.tempdir) 1284 else: 1285 bb.utils.prunedir(self.tempdir) 1286 1287 @skipIfNoSvn() 1288 @skipIfNoNetwork() 1289 def test_noexternal_svn(self): 1290 # Always match the rev count from setUp (currently rev 2) 1291 url = "svn://%s;module=trunk;protocol=file;rev=2" % self.repo_url.replace('file://', '') 1292 fetcher = bb.fetch.Fetch([url], self.d) 1293 fetcher.download() 1294 os.chdir(os.path.dirname(self.unpackdir)) 1295 fetcher.unpack(self.unpackdir) 1296 1297 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") 1298 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") 1299 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should NOT exist") 1300 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should NOT exit") 1301 1302 @skipIfNoSvn() 1303 def test_external_svn(self): 1304 # Always match the rev count from setUp (currently rev 2) 1305 url = "svn://%s;module=trunk;protocol=file;externals=allowed;rev=2" % self.repo_url.replace('file://', '') 1306 fetcher = bb.fetch.Fetch([url], self.d) 1307 fetcher.download() 1308 os.chdir(os.path.dirname(self.unpackdir)) 1309 fetcher.unpack(self.unpackdir) 1310 1311 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") 1312 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") 1313 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should exist") 1314 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should exit") 1315 1316class TrustedNetworksTest(FetcherTest): 1317 def test_trusted_network(self): 1318 # Ensure trusted_network returns False when the host IS in the list. 1319 url = "git://Someserver.org/foo;rev=1;branch=master" 1320 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org") 1321 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1322 1323 def test_wild_trusted_network(self): 1324 # Ensure trusted_network returns true when the *.host IS in the list. 1325 url = "git://Someserver.org/foo;rev=1;branch=master" 1326 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1327 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1328 1329 def test_prefix_wild_trusted_network(self): 1330 # Ensure trusted_network returns true when the prefix matches *.host. 1331 url = "git://git.Someserver.org/foo;rev=1;branch=master" 1332 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1333 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1334 1335 def test_two_prefix_wild_trusted_network(self): 1336 # Ensure trusted_network returns true when the prefix matches *.host. 1337 url = "git://something.git.Someserver.org/foo;rev=1;branch=master" 1338 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1339 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1340 1341 def test_port_trusted_network(self): 1342 # Ensure trusted_network returns True, even if the url specifies a port. 1343 url = "git://someserver.org:8080/foo;rev=1;branch=master" 1344 self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org") 1345 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1346 1347 def test_untrusted_network(self): 1348 # Ensure trusted_network returns False when the host is NOT in the list. 1349 url = "git://someserver.org/foo;rev=1;branch=master" 1350 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org") 1351 self.assertFalse(bb.fetch.trusted_network(self.d, url)) 1352 1353 def test_wild_untrusted_network(self): 1354 # Ensure trusted_network returns False when the host is NOT in the list. 1355 url = "git://*.someserver.org/foo;rev=1;branch=master" 1356 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org") 1357 self.assertFalse(bb.fetch.trusted_network(self.d, url)) 1358 1359class URLHandle(unittest.TestCase): 1360 1361 datatable = { 1362 "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}), 1363 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}), 1364 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])), 1365 "git://git.openembedded.org/bitbake;branch=@foo;protocol=https" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo', 'protocol' : 'https'}), 1366 "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}), 1367 "https://somesite.com/somerepo.git;user=anyUser:idtoken=1234" : ('https', 'somesite.com', '/somerepo.git', '', '', {'user': 'anyUser:idtoken=1234'}), 1368 r'git://s.o-me_ONE:!#$%^&*()-_={}[]\|:?,.<>~`@git.openembedded.org/bitbake;branch=main;protocol=https': ('git', 'git.openembedded.org', '/bitbake', 's.o-me_ONE', r'!#$%^&*()-_={}[]\|:?,.<>~`', {'branch': 'main', 'protocol' : 'https'}), 1369 } 1370 # we require a pathname to encodeurl but users can still pass such urls to 1371 # decodeurl and we need to handle them 1372 decodedata = datatable.copy() 1373 decodedata.update({ 1374 "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}), 1375 "npmsw://some.registry.url;package=@pkg;version=latest": ('npmsw', 'some.registry.url', '/', '', '', {'package': '@pkg', 'version': 'latest'}), 1376 }) 1377 1378 def test_decodeurl(self): 1379 for k, v in self.decodedata.items(): 1380 result = bb.fetch.decodeurl(k) 1381 self.assertEqual(result, v) 1382 1383 def test_encodeurl(self): 1384 for k, v in self.datatable.items(): 1385 result = bb.fetch.encodeurl(v) 1386 self.assertEqual(result, k) 1387 1388class FetchLatestVersionTest(FetcherTest): 1389 1390 test_git_uris = { 1391 # version pattern "X.Y.Z" 1392 ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "", "") 1393 : "1.99.4", 1394 # version pattern "vX.Y" 1395 # mirror of git.infradead.org since network issues interfered with testing 1396 ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git;branch=master;protocol=https", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "", "") 1397 : "1.5.0", 1398 # version pattern "pkg_name-X.Y" 1399 # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing 1400 ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto;branch=master;protocol=https", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "", "") 1401 : "1.0", 1402 # version pattern "pkg_name-vX.Y.Z" 1403 ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git;branch=master;protocol=https", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "", "") 1404 : "1.4.0", 1405 # combination version pattern 1406 ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "", "") 1407 : "1.2.0", 1408 ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "", "") 1409 : "2014.01", 1410 # version pattern "yyyymmdd" 1411 ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "", "") 1412 : "20120614", 1413 # packages with a valid UPSTREAM_CHECK_GITTAGREGEX 1414 # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing 1415 ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap;branch=master;protocol=https", "ae0394e687f1a77e966cf72f895da91840dffb8f", r"(?P<pver>(\d+\.(\d\.?)*))", "") 1416 : "0.4.3", 1417 ("build-appliance-image", "git://git.yoctoproject.org/poky;branch=master;protocol=https", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P<pver>(([0-9][\.|_]?)+[0-9]))", "") 1418 : "11.0.0", 1419 ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))", "") 1420 : "1.3.59", 1421 ("remake", "git://github.com/rocky/remake.git;protocol=https;branch=master", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))", "") 1422 : "3.82+dbg0.9", 1423 ("sysdig", "git://github.com/draios/sysdig.git;branch=dev;protocol=https", "4fb6288275f567f63515df0ff0a6518043ecfa9b", r"^(?P<pver>\d+(\.\d+)+)", "10.0.0") 1424 : "0.28.0", 1425 } 1426 1427 test_wget_uris = { 1428 # 1429 # packages with versions inside directory name 1430 # 1431 # http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2 1432 ("util-linux", "/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2", "", "") 1433 : "2.24.2", 1434 # http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz 1435 ("enchant", "/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz", "", "") 1436 : "1.6.0", 1437 # http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz 1438 ("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz", "", "") 1439 : "2.8.12.1", 1440 # https://download.gnome.org/sources/libxml2/2.9/libxml2-2.9.14.tar.xz 1441 ("libxml2", "/software/libxml2/2.9/libxml2-2.9.14.tar.xz", "", "") 1442 : "2.10.3", 1443 # 1444 # packages with versions only in current directory 1445 # 1446 # https://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2 1447 ("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "") 1448 : "2.19", 1449 # https://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2 1450 ("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "") 1451 : "20120814", 1452 # 1453 # packages with "99" in the name of possible version 1454 # 1455 # http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz 1456 ("pulseaudio", "/software/pulseaudio/releases/pulseaudio-4.0.tar.xz", "", "") 1457 : "5.0", 1458 # http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2 1459 ("xserver-xorg", "/releases/individual/xserver/xorg-server-1.15.1.tar.bz2", "", "") 1460 : "1.15.1", 1461 # 1462 # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX 1463 # 1464 # http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2 1465 # https://github.com/apple/cups/releases 1466 ("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", "/apple/cups/releases", r"(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz") 1467 : "2.0.0", 1468 # http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz 1469 # http://ftp.debian.org/debian/pool/main/d/db5.3/ 1470 ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", r"(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz") 1471 : "5.3.10", 1472 # 1473 # packages where the tarball compression changed in the new version 1474 # 1475 # http://ftp.debian.org/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz 1476 ("minicom", "/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz", "", "") 1477 : "2.8", 1478 } 1479 1480 @skipIfNoNetwork() 1481 def test_git_latest_versionstring(self): 1482 for k, v in self.test_git_uris.items(): 1483 self.d.setVar("PN", k[0]) 1484 self.d.setVar("SRCREV", k[2]) 1485 self.d.setVar("UPSTREAM_CHECK_GITTAGREGEX", k[3]) 1486 ud = bb.fetch2.FetchData(k[1], self.d) 1487 pupver= ud.method.latest_versionstring(ud, self.d) 1488 verstring = pupver[0] 1489 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) 1490 r = bb.utils.vercmp_string(v, verstring) 1491 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) 1492 if k[4]: 1493 r = bb.utils.vercmp_string(verstring, k[4]) 1494 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], verstring, k[4])) 1495 1496 def test_wget_latest_versionstring(self): 1497 testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata" 1498 server = HTTPService(testdata) 1499 server.start() 1500 port = server.port 1501 try: 1502 for k, v in self.test_wget_uris.items(): 1503 self.d.setVar("PN", k[0]) 1504 checkuri = "" 1505 if k[2]: 1506 checkuri = "http://localhost:%s/" % port + k[2] 1507 self.d.setVar("UPSTREAM_CHECK_URI", checkuri) 1508 self.d.setVar("UPSTREAM_CHECK_REGEX", k[3]) 1509 url = "http://localhost:%s/" % port + k[1] 1510 ud = bb.fetch2.FetchData(url, self.d) 1511 pupver = ud.method.latest_versionstring(ud, self.d) 1512 verstring = pupver[0] 1513 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) 1514 r = bb.utils.vercmp_string(v, verstring) 1515 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) 1516 finally: 1517 server.stop() 1518 1519 1520class FetchCheckStatusTest(FetcherTest): 1521 test_wget_uris = ["https://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz", 1522 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz", 1523 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz", 1524 "https://yoctoproject.org/", 1525 "https://docs.yoctoproject.org", 1526 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz", 1527 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz", 1528 "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz", 1529 # GitHub releases are hosted on Amazon S3, which doesn't support HEAD 1530 "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz" 1531 ] 1532 1533 @skipIfNoNetwork() 1534 def test_wget_checkstatus(self): 1535 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d) 1536 for u in self.test_wget_uris: 1537 with self.subTest(url=u): 1538 ud = fetch.ud[u] 1539 m = ud.method 1540 ret = m.checkstatus(fetch, ud, self.d) 1541 self.assertTrue(ret, msg="URI %s, can't check status" % (u)) 1542 1543 @skipIfNoNetwork() 1544 def test_wget_checkstatus_connection_cache(self): 1545 from bb.fetch2 import FetchConnectionCache 1546 1547 connection_cache = FetchConnectionCache() 1548 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d, 1549 connection_cache = connection_cache) 1550 1551 for u in self.test_wget_uris: 1552 with self.subTest(url=u): 1553 ud = fetch.ud[u] 1554 m = ud.method 1555 ret = m.checkstatus(fetch, ud, self.d) 1556 self.assertTrue(ret, msg="URI %s, can't check status" % (u)) 1557 1558 connection_cache.close_connections() 1559 1560 1561class GitMakeShallowTest(FetcherTest): 1562 def setUp(self): 1563 FetcherTest.setUp(self) 1564 self.gitdir = os.path.join(self.tempdir, 'gitshallow') 1565 bb.utils.mkdirhier(self.gitdir) 1566 self.git_init() 1567 1568 def assertRefs(self, expected_refs): 1569 actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines() 1570 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs).splitlines() 1571 self.assertEqual(sorted(full_expected), sorted(actual_refs)) 1572 1573 def assertRevCount(self, expected_count, args=None): 1574 if args is None: 1575 args = ['HEAD'] 1576 revs = self.git(['rev-list'] + args) 1577 actual_count = len(revs.splitlines()) 1578 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count)) 1579 1580 def make_shallow(self, args=None): 1581 if args is None: 1582 args = ['HEAD'] 1583 return bb.process.run([bb.fetch2.git.Git.make_shallow_path] + args, cwd=self.gitdir) 1584 1585 def add_empty_file(self, path, msg=None): 1586 if msg is None: 1587 msg = path 1588 open(os.path.join(self.gitdir, path), 'w').close() 1589 self.git(['add', path]) 1590 self.git(['commit', '-m', msg, path]) 1591 1592 def test_make_shallow_single_branch_no_merge(self): 1593 self.add_empty_file('a') 1594 self.add_empty_file('b') 1595 self.assertRevCount(2) 1596 self.make_shallow() 1597 self.assertRevCount(1) 1598 1599 def test_make_shallow_single_branch_one_merge(self): 1600 self.add_empty_file('a') 1601 self.add_empty_file('b') 1602 self.git('checkout -b a_branch') 1603 self.add_empty_file('c') 1604 self.git('checkout master') 1605 self.add_empty_file('d') 1606 self.git('merge --no-ff --no-edit a_branch') 1607 self.git('branch -d a_branch') 1608 self.add_empty_file('e') 1609 self.assertRevCount(6) 1610 self.make_shallow(['HEAD~2']) 1611 self.assertRevCount(5) 1612 1613 def test_make_shallow_at_merge(self): 1614 self.add_empty_file('a') 1615 self.git('checkout -b a_branch') 1616 self.add_empty_file('b') 1617 self.git('checkout master') 1618 self.git('merge --no-ff --no-edit a_branch') 1619 self.git('branch -d a_branch') 1620 self.assertRevCount(3) 1621 self.make_shallow() 1622 self.assertRevCount(1) 1623 1624 def test_make_shallow_annotated_tag(self): 1625 self.add_empty_file('a') 1626 self.add_empty_file('b') 1627 self.git('tag -a -m a_tag a_tag') 1628 self.assertRevCount(2) 1629 self.make_shallow(['a_tag']) 1630 self.assertRevCount(1) 1631 1632 def test_make_shallow_multi_ref(self): 1633 self.add_empty_file('a') 1634 self.add_empty_file('b') 1635 self.git('checkout -b a_branch') 1636 self.add_empty_file('c') 1637 self.git('checkout master') 1638 self.add_empty_file('d') 1639 self.git('checkout -b a_branch_2') 1640 self.add_empty_file('a_tag') 1641 self.git('tag a_tag') 1642 self.git('checkout master') 1643 self.git('branch -D a_branch_2') 1644 self.add_empty_file('e') 1645 self.assertRevCount(6, ['--all']) 1646 self.make_shallow() 1647 self.assertRevCount(5, ['--all']) 1648 1649 def test_make_shallow_multi_ref_trim(self): 1650 self.add_empty_file('a') 1651 self.git('checkout -b a_branch') 1652 self.add_empty_file('c') 1653 self.git('checkout master') 1654 self.assertRevCount(1) 1655 self.assertRevCount(2, ['--all']) 1656 self.assertRefs(['master', 'a_branch']) 1657 self.make_shallow(['-r', 'master', 'HEAD']) 1658 self.assertRevCount(1, ['--all']) 1659 self.assertRefs(['master']) 1660 1661 def test_make_shallow_noop(self): 1662 self.add_empty_file('a') 1663 self.assertRevCount(1) 1664 self.make_shallow() 1665 self.assertRevCount(1) 1666 1667 @skipIfNoNetwork() 1668 def test_make_shallow_bitbake(self): 1669 self.git('remote add origin https://github.com/openembedded/bitbake') 1670 self.git('fetch --tags origin') 1671 orig_revs = len(self.git('rev-list --all').splitlines()) 1672 self.make_shallow(['refs/tags/1.10.0']) 1673 self.assertRevCount(orig_revs - 1746, ['--all']) 1674 1675class GitShallowTest(FetcherTest): 1676 def setUp(self): 1677 FetcherTest.setUp(self) 1678 self.gitdir = os.path.join(self.tempdir, 'git') 1679 self.srcdir = os.path.join(self.tempdir, 'gitsource') 1680 1681 bb.utils.mkdirhier(self.srcdir) 1682 self.git_init(cwd=self.srcdir) 1683 self.d.setVar('WORKDIR', self.tempdir) 1684 self.d.setVar('S', self.gitdir) 1685 self.d.delVar('PREMIRRORS') 1686 self.d.delVar('MIRRORS') 1687 1688 uri = 'git://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 1689 self.d.setVar('SRC_URI', uri) 1690 self.d.setVar('SRCREV', '${AUTOREV}') 1691 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') 1692 1693 self.d.setVar('BB_GIT_SHALLOW', '1') 1694 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0') 1695 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 1696 self.d.setVar("__BBSRCREV_SEEN", "1") 1697 1698 def assertRefs(self, expected_refs, cwd=None): 1699 if cwd is None: 1700 cwd = self.gitdir 1701 actual_refs = self.git(['for-each-ref', '--format=%(refname)'], cwd=cwd).splitlines() 1702 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs, cwd=cwd).splitlines() 1703 self.assertEqual(sorted(set(full_expected)), sorted(set(actual_refs))) 1704 1705 def assertRevCount(self, expected_count, args=None, cwd=None): 1706 if args is None: 1707 args = ['HEAD'] 1708 if cwd is None: 1709 cwd = self.gitdir 1710 revs = self.git(['rev-list'] + args, cwd=cwd) 1711 actual_count = len(revs.splitlines()) 1712 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count)) 1713 1714 def add_empty_file(self, path, cwd=None, msg=None): 1715 if msg is None: 1716 msg = path 1717 if cwd is None: 1718 cwd = self.srcdir 1719 open(os.path.join(cwd, path), 'w').close() 1720 self.git(['add', path], cwd) 1721 self.git(['commit', '-m', msg, path], cwd) 1722 1723 def fetch(self, uri=None): 1724 if uri is None: 1725 uris = self.d.getVar('SRC_URI').split() 1726 uri = uris[0] 1727 d = self.d 1728 else: 1729 d = self.d.createCopy() 1730 d.setVar('SRC_URI', uri) 1731 uri = d.expand(uri) 1732 uris = [uri] 1733 1734 fetcher = bb.fetch2.Fetch(uris, d) 1735 fetcher.download() 1736 ud = fetcher.ud[uri] 1737 return fetcher, ud 1738 1739 def fetch_and_unpack(self, uri=None): 1740 fetcher, ud = self.fetch(uri) 1741 fetcher.unpack(self.d.getVar('WORKDIR')) 1742 assert os.path.exists(self.d.getVar('S')) 1743 return fetcher, ud 1744 1745 def fetch_shallow(self, uri=None, disabled=False, keepclone=False): 1746 """Fetch a uri, generating a shallow tarball, then unpack using it""" 1747 fetcher, ud = self.fetch_and_unpack(uri) 1748 assert os.path.exists(ud.clonedir), 'Git clone in DLDIR (%s) does not exist for uri %s' % (ud.clonedir, uri) 1749 1750 # Confirm that the unpacked repo is unshallow 1751 if not disabled: 1752 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0])) 1753 1754 # fetch and unpack, from the shallow tarball 1755 bb.utils.remove(self.gitdir, recurse=True) 1756 bb.process.run('chmod u+w -R "%s"' % ud.clonedir) 1757 bb.utils.remove(ud.clonedir, recurse=True) 1758 bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True) 1759 1760 # confirm that the unpacked repo is used when no git clone or git 1761 # mirror tarball is available 1762 fetcher, ud = self.fetch_and_unpack(uri) 1763 if not disabled: 1764 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is not shallow' % self.gitdir 1765 else: 1766 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is shallow' % self.gitdir 1767 return fetcher, ud 1768 1769 def test_shallow_disabled(self): 1770 self.add_empty_file('a') 1771 self.add_empty_file('b') 1772 self.assertRevCount(2, cwd=self.srcdir) 1773 1774 self.d.setVar('BB_GIT_SHALLOW', '0') 1775 self.fetch_shallow(disabled=True) 1776 self.assertRevCount(2) 1777 1778 def test_shallow_nobranch(self): 1779 self.add_empty_file('a') 1780 self.add_empty_file('b') 1781 self.assertRevCount(2, cwd=self.srcdir) 1782 1783 srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip() 1784 self.d.setVar('SRCREV', srcrev) 1785 uri = self.d.getVar('SRC_URI').split()[0] 1786 uri = '%s;nobranch=1;bare=1' % uri 1787 1788 self.fetch_shallow(uri) 1789 self.assertRevCount(1) 1790 1791 # shallow refs are used to ensure the srcrev sticks around when we 1792 # have no other branches referencing it 1793 self.assertRefs(['refs/shallow/default']) 1794 1795 def test_shallow_default_depth_1(self): 1796 # Create initial git repo 1797 self.add_empty_file('a') 1798 self.add_empty_file('b') 1799 self.assertRevCount(2, cwd=self.srcdir) 1800 1801 self.fetch_shallow() 1802 self.assertRevCount(1) 1803 1804 def test_shallow_depth_0_disables(self): 1805 self.add_empty_file('a') 1806 self.add_empty_file('b') 1807 self.assertRevCount(2, cwd=self.srcdir) 1808 1809 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 1810 self.fetch_shallow(disabled=True) 1811 self.assertRevCount(2) 1812 1813 def test_shallow_depth_default_override(self): 1814 self.add_empty_file('a') 1815 self.add_empty_file('b') 1816 self.assertRevCount(2, cwd=self.srcdir) 1817 1818 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '2') 1819 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '1') 1820 self.fetch_shallow() 1821 self.assertRevCount(1) 1822 1823 def test_shallow_depth_default_override_disable(self): 1824 self.add_empty_file('a') 1825 self.add_empty_file('b') 1826 self.add_empty_file('c') 1827 self.assertRevCount(3, cwd=self.srcdir) 1828 1829 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 1830 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '2') 1831 self.fetch_shallow() 1832 self.assertRevCount(2) 1833 1834 def test_current_shallow_out_of_date_clone(self): 1835 # Create initial git repo 1836 self.add_empty_file('a') 1837 self.add_empty_file('b') 1838 self.add_empty_file('c') 1839 self.assertRevCount(3, cwd=self.srcdir) 1840 1841 # Clone and generate mirror tarball 1842 fetcher, ud = self.fetch() 1843 1844 # Ensure we have a current mirror tarball, but an out of date clone 1845 self.git('update-ref refs/heads/master refs/heads/master~1', cwd=ud.clonedir) 1846 self.assertRevCount(2, cwd=ud.clonedir) 1847 1848 # Fetch and unpack, from the current tarball, not the out of date clone 1849 bb.utils.remove(self.gitdir, recurse=True) 1850 fetcher, ud = self.fetch() 1851 fetcher.unpack(self.d.getVar('WORKDIR')) 1852 self.assertRevCount(1) 1853 1854 def test_shallow_single_branch_no_merge(self): 1855 self.add_empty_file('a') 1856 self.add_empty_file('b') 1857 self.assertRevCount(2, cwd=self.srcdir) 1858 1859 self.fetch_shallow() 1860 self.assertRevCount(1) 1861 assert os.path.exists(os.path.join(self.gitdir, 'a')) 1862 assert os.path.exists(os.path.join(self.gitdir, 'b')) 1863 1864 def test_shallow_no_dangling(self): 1865 self.add_empty_file('a') 1866 self.add_empty_file('b') 1867 self.assertRevCount(2, cwd=self.srcdir) 1868 1869 self.fetch_shallow() 1870 self.assertRevCount(1) 1871 assert not self.git('fsck --dangling') 1872 1873 def test_shallow_srcrev_branch_truncation(self): 1874 self.add_empty_file('a') 1875 self.add_empty_file('b') 1876 b_commit = self.git('rev-parse HEAD', cwd=self.srcdir).rstrip() 1877 self.add_empty_file('c') 1878 self.assertRevCount(3, cwd=self.srcdir) 1879 1880 self.d.setVar('SRCREV', b_commit) 1881 self.fetch_shallow() 1882 1883 # The 'c' commit was removed entirely, and 'a' was removed from history 1884 self.assertRevCount(1, ['--all']) 1885 self.assertEqual(self.git('rev-parse HEAD').strip(), b_commit) 1886 assert os.path.exists(os.path.join(self.gitdir, 'a')) 1887 assert os.path.exists(os.path.join(self.gitdir, 'b')) 1888 assert not os.path.exists(os.path.join(self.gitdir, 'c')) 1889 1890 def test_shallow_ref_pruning(self): 1891 self.add_empty_file('a') 1892 self.add_empty_file('b') 1893 self.git('branch a_branch', cwd=self.srcdir) 1894 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir) 1895 self.assertRevCount(2, cwd=self.srcdir) 1896 1897 self.fetch_shallow() 1898 1899 self.assertRefs(['master', 'origin/master']) 1900 self.assertRevCount(1) 1901 1902 def test_shallow_submodules(self): 1903 self.add_empty_file('a') 1904 self.add_empty_file('b') 1905 1906 smdir = os.path.join(self.tempdir, 'gitsubmodule') 1907 bb.utils.mkdirhier(smdir) 1908 self.git_init(cwd=smdir) 1909 # Make this look like it was cloned from a remote... 1910 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) 1911 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) 1912 self.add_empty_file('asub', cwd=smdir) 1913 self.add_empty_file('bsub', cwd=smdir) 1914 1915 self.git('submodule init', cwd=self.srcdir) 1916 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir) 1917 self.git('submodule update', cwd=self.srcdir) 1918 self.git('commit -m submodule -a', cwd=self.srcdir) 1919 1920 uri = 'gitsm://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 1921 fetcher, ud = self.fetch_shallow(uri) 1922 1923 # Verify the main repository is shallow 1924 self.assertRevCount(1) 1925 1926 # Verify the gitsubmodule directory is present 1927 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule')) 1928 1929 # Verify the submodule is also shallow 1930 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule')) 1931 1932 def test_shallow_submodule_mirrors(self): 1933 self.add_empty_file('a') 1934 self.add_empty_file('b') 1935 1936 smdir = os.path.join(self.tempdir, 'gitsubmodule') 1937 bb.utils.mkdirhier(smdir) 1938 self.git_init(cwd=smdir) 1939 # Make this look like it was cloned from a remote... 1940 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) 1941 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) 1942 self.add_empty_file('asub', cwd=smdir) 1943 self.add_empty_file('bsub', cwd=smdir) 1944 1945 self.git('submodule init', cwd=self.srcdir) 1946 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir) 1947 self.git('submodule update', cwd=self.srcdir) 1948 self.git('commit -m submodule -a', cwd=self.srcdir) 1949 1950 uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir 1951 1952 # Fetch once to generate the shallow tarball 1953 fetcher, ud = self.fetch(uri) 1954 1955 # Set up the mirror 1956 mirrordir = os.path.join(self.tempdir, 'mirror') 1957 bb.utils.rename(self.dldir, mirrordir) 1958 self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/' % mirrordir) 1959 1960 # Fetch from the mirror 1961 bb.utils.remove(self.dldir, recurse=True) 1962 bb.utils.remove(self.gitdir, recurse=True) 1963 self.fetch_and_unpack(uri) 1964 1965 # Verify the main repository is shallow 1966 self.assertRevCount(1) 1967 1968 # Verify the gitsubmodule directory is present 1969 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule')) 1970 1971 # Verify the submodule is also shallow 1972 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule')) 1973 1974 if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')): 1975 def test_shallow_annex(self): 1976 self.add_empty_file('a') 1977 self.add_empty_file('b') 1978 self.git('annex init', cwd=self.srcdir) 1979 open(os.path.join(self.srcdir, 'c'), 'w').close() 1980 self.git('annex add c', cwd=self.srcdir) 1981 self.git('commit --author "Foo Bar <foo@bar>" -m annex-c -a', cwd=self.srcdir) 1982 bb.process.run('chmod u+w -R %s' % self.srcdir) 1983 1984 uri = 'gitannex://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 1985 fetcher, ud = self.fetch_shallow(uri) 1986 1987 self.assertRevCount(1) 1988 assert './.git/annex/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0] 1989 assert os.path.exists(os.path.join(self.gitdir, 'c')) 1990 1991 def test_shallow_multi_one_uri(self): 1992 # Create initial git repo 1993 self.add_empty_file('a') 1994 self.add_empty_file('b') 1995 self.git('checkout -b a_branch', cwd=self.srcdir) 1996 self.add_empty_file('c') 1997 self.add_empty_file('d') 1998 self.git('checkout master', cwd=self.srcdir) 1999 self.git('tag v0.0 a_branch', cwd=self.srcdir) 2000 self.add_empty_file('e') 2001 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 2002 self.add_empty_file('f') 2003 self.assertRevCount(7, cwd=self.srcdir) 2004 2005 uri = self.d.getVar('SRC_URI').split()[0] 2006 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri 2007 2008 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2009 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2010 self.d.setVar('SRCREV_master', '${AUTOREV}') 2011 self.d.setVar('SRCREV_a_branch', '${AUTOREV}') 2012 2013 self.fetch_shallow(uri) 2014 2015 self.assertRevCount(5) 2016 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 2017 2018 def test_shallow_multi_one_uri_depths(self): 2019 # Create initial git repo 2020 self.add_empty_file('a') 2021 self.add_empty_file('b') 2022 self.git('checkout -b a_branch', cwd=self.srcdir) 2023 self.add_empty_file('c') 2024 self.add_empty_file('d') 2025 self.git('checkout master', cwd=self.srcdir) 2026 self.add_empty_file('e') 2027 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 2028 self.add_empty_file('f') 2029 self.assertRevCount(7, cwd=self.srcdir) 2030 2031 uri = self.d.getVar('SRC_URI').split()[0] 2032 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri 2033 2034 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2035 self.d.setVar('BB_GIT_SHALLOW_DEPTH_master', '3') 2036 self.d.setVar('BB_GIT_SHALLOW_DEPTH_a_branch', '1') 2037 self.d.setVar('SRCREV_master', '${AUTOREV}') 2038 self.d.setVar('SRCREV_a_branch', '${AUTOREV}') 2039 2040 self.fetch_shallow(uri) 2041 2042 self.assertRevCount(4, ['--all']) 2043 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 2044 2045 def test_shallow_clone_preferred_over_shallow(self): 2046 self.add_empty_file('a') 2047 self.add_empty_file('b') 2048 2049 # Fetch once to generate the shallow tarball 2050 fetcher, ud = self.fetch() 2051 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0])) 2052 2053 # Fetch and unpack with both the clonedir and shallow tarball available 2054 bb.utils.remove(self.gitdir, recurse=True) 2055 fetcher, ud = self.fetch_and_unpack() 2056 2057 # The unpacked tree should *not* be shallow 2058 self.assertRevCount(2) 2059 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')) 2060 2061 def test_shallow_mirrors(self): 2062 self.add_empty_file('a') 2063 self.add_empty_file('b') 2064 2065 # Fetch once to generate the shallow tarball 2066 fetcher, ud = self.fetch() 2067 mirrortarball = ud.mirrortarballs[0] 2068 assert os.path.exists(os.path.join(self.dldir, mirrortarball)) 2069 2070 # Set up the mirror 2071 mirrordir = os.path.join(self.tempdir, 'mirror') 2072 bb.utils.mkdirhier(mirrordir) 2073 self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/' % mirrordir) 2074 2075 bb.utils.rename(os.path.join(self.dldir, mirrortarball), 2076 os.path.join(mirrordir, mirrortarball)) 2077 2078 # Fetch from the mirror 2079 bb.utils.remove(self.dldir, recurse=True) 2080 bb.utils.remove(self.gitdir, recurse=True) 2081 self.fetch_and_unpack() 2082 self.assertRevCount(1) 2083 2084 def test_shallow_invalid_depth(self): 2085 self.add_empty_file('a') 2086 self.add_empty_file('b') 2087 2088 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '-12') 2089 with self.assertRaises(bb.fetch2.FetchError): 2090 self.fetch() 2091 2092 def test_shallow_invalid_depth_default(self): 2093 self.add_empty_file('a') 2094 self.add_empty_file('b') 2095 2096 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '-12') 2097 with self.assertRaises(bb.fetch2.FetchError): 2098 self.fetch() 2099 2100 def test_shallow_extra_refs(self): 2101 self.add_empty_file('a') 2102 self.add_empty_file('b') 2103 self.git('branch a_branch', cwd=self.srcdir) 2104 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir) 2105 self.assertRevCount(2, cwd=self.srcdir) 2106 2107 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/a_branch') 2108 self.fetch_shallow() 2109 2110 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 2111 self.assertRevCount(1) 2112 2113 def test_shallow_extra_refs_wildcard(self): 2114 self.add_empty_file('a') 2115 self.add_empty_file('b') 2116 self.git('branch a_branch', cwd=self.srcdir) 2117 self.git('tag v1.0', cwd=self.srcdir) 2118 self.assertRefs(['master', 'a_branch', 'v1.0'], cwd=self.srcdir) 2119 self.assertRevCount(2, cwd=self.srcdir) 2120 2121 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*') 2122 self.fetch_shallow() 2123 2124 self.assertRefs(['master', 'origin/master', 'v1.0']) 2125 self.assertRevCount(1) 2126 2127 def test_shallow_missing_extra_refs(self): 2128 self.add_empty_file('a') 2129 self.add_empty_file('b') 2130 2131 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/foo') 2132 with self.assertRaises(bb.fetch2.FetchError): 2133 self.fetch() 2134 2135 def test_shallow_missing_extra_refs_wildcard(self): 2136 self.add_empty_file('a') 2137 self.add_empty_file('b') 2138 2139 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*') 2140 self.fetch() 2141 2142 def test_shallow_remove_revs(self): 2143 # Create initial git repo 2144 self.add_empty_file('a') 2145 self.add_empty_file('b') 2146 self.git('checkout -b a_branch', cwd=self.srcdir) 2147 self.add_empty_file('c') 2148 self.add_empty_file('d') 2149 self.git('checkout master', cwd=self.srcdir) 2150 self.git('tag v0.0 a_branch', cwd=self.srcdir) 2151 self.add_empty_file('e') 2152 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 2153 self.git('branch -d a_branch', cwd=self.srcdir) 2154 self.add_empty_file('f') 2155 self.assertRevCount(7, cwd=self.srcdir) 2156 2157 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2158 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2159 2160 self.fetch_shallow() 2161 2162 self.assertRevCount(5) 2163 2164 def test_shallow_invalid_revs(self): 2165 self.add_empty_file('a') 2166 self.add_empty_file('b') 2167 2168 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2169 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2170 2171 with self.assertRaises(bb.fetch2.FetchError): 2172 self.fetch() 2173 2174 def test_shallow_fetch_missing_revs(self): 2175 self.add_empty_file('a') 2176 self.add_empty_file('b') 2177 fetcher, ud = self.fetch(self.d.getVar('SRC_URI')) 2178 self.git('tag v0.0 master', cwd=self.srcdir) 2179 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2180 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2181 self.fetch_shallow() 2182 2183 def test_shallow_fetch_missing_revs_fails(self): 2184 self.add_empty_file('a') 2185 self.add_empty_file('b') 2186 fetcher, ud = self.fetch(self.d.getVar('SRC_URI')) 2187 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2188 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2189 2190 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm: 2191 self.fetch_shallow() 2192 self.assertIn("Unable to find revision v0.0 even from upstream", cm.output[0]) 2193 2194 @skipIfNoNetwork() 2195 def test_bitbake(self): 2196 self.git('remote add --mirror=fetch origin https://github.com/openembedded/bitbake', cwd=self.srcdir) 2197 self.git('config core.bare true', cwd=self.srcdir) 2198 self.git('fetch', cwd=self.srcdir) 2199 2200 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2201 # Note that the 1.10.0 tag is annotated, so this also tests 2202 # reference of an annotated vs unannotated tag 2203 self.d.setVar('BB_GIT_SHALLOW_REVS', '1.10.0') 2204 2205 self.fetch_shallow() 2206 2207 # Confirm that the history of 1.10.0 was removed 2208 orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines()) 2209 revs = len(self.git('rev-list master').splitlines()) 2210 self.assertNotEqual(orig_revs, revs) 2211 self.assertRefs(['master', 'origin/master']) 2212 self.assertRevCount(orig_revs - 1758) 2213 2214 def test_that_unpack_throws_an_error_when_the_git_clone_nor_shallow_tarball_exist(self): 2215 self.add_empty_file('a') 2216 fetcher, ud = self.fetch() 2217 bb.utils.remove(self.gitdir, recurse=True) 2218 bb.utils.remove(self.dldir, recurse=True) 2219 2220 with self.assertRaises(bb.fetch2.UnpackError) as context: 2221 fetcher.unpack(self.d.getVar('WORKDIR')) 2222 2223 self.assertIn("No up to date source found", context.exception.msg) 2224 self.assertIn("clone directory not available or not up to date", context.exception.msg) 2225 2226 @skipIfNoNetwork() 2227 def test_that_unpack_does_work_when_using_git_shallow_tarball_but_tarball_is_not_available(self): 2228 self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0') 2229 self.d.setVar('BB_GIT_SHALLOW', '1') 2230 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 2231 fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests;branch=master;protocol=https"], self.d) 2232 fetcher.download() 2233 2234 bb.utils.remove(self.dldir + "/*.tar.gz") 2235 fetcher.unpack(self.unpackdir) 2236 2237 dir = os.listdir(self.unpackdir + "/git/") 2238 self.assertIn("fstests.doap", dir) 2239 2240class GitLfsTest(FetcherTest): 2241 def skipIfNoGitLFS(): 2242 import shutil 2243 if not shutil.which('git-lfs'): 2244 return unittest.skip('git-lfs not installed') 2245 return lambda f: f 2246 2247 def setUp(self): 2248 FetcherTest.setUp(self) 2249 2250 self.gitdir = os.path.join(self.tempdir, 'git') 2251 self.srcdir = os.path.join(self.tempdir, 'gitsource') 2252 2253 self.d.setVar('WORKDIR', self.tempdir) 2254 self.d.setVar('S', self.gitdir) 2255 self.d.delVar('PREMIRRORS') 2256 self.d.delVar('MIRRORS') 2257 2258 self.d.setVar('SRCREV', '${AUTOREV}') 2259 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') 2260 self.d.setVar("__BBSRCREV_SEEN", "1") 2261 2262 bb.utils.mkdirhier(self.srcdir) 2263 self.git_init(cwd=self.srcdir) 2264 with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs: 2265 attrs.write('*.mp3 filter=lfs -text') 2266 self.git(['add', '.gitattributes'], cwd=self.srcdir) 2267 self.git(['commit', '-m', "attributes", '.gitattributes'], cwd=self.srcdir) 2268 2269 def fetch(self, uri=None, download=True): 2270 uris = self.d.getVar('SRC_URI').split() 2271 uri = uris[0] 2272 d = self.d 2273 2274 fetcher = bb.fetch2.Fetch(uris, d) 2275 if download: 2276 fetcher.download() 2277 ud = fetcher.ud[uri] 2278 return fetcher, ud 2279 2280 def get_real_git_lfs_file(self): 2281 self.d.setVar('PATH', os.environ.get('PATH')) 2282 fetcher, ud = self.fetch() 2283 fetcher.unpack(self.d.getVar('WORKDIR')) 2284 unpacked_lfs_file = os.path.join(self.d.getVar('WORKDIR'), 'git', "Cat_poster_1.jpg") 2285 return unpacked_lfs_file 2286 2287 @skipIfNoGitLFS() 2288 @skipIfNoNetwork() 2289 def test_real_git_lfs_repo_succeeds_without_lfs_param(self): 2290 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master") 2291 f = self.get_real_git_lfs_file() 2292 self.assertTrue(os.path.exists(f)) 2293 self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f)) 2294 2295 @skipIfNoGitLFS() 2296 @skipIfNoNetwork() 2297 def test_real_git_lfs_repo_succeeds(self): 2298 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=1") 2299 f = self.get_real_git_lfs_file() 2300 self.assertTrue(os.path.exists(f)) 2301 self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f)) 2302 2303 @skipIfNoGitLFS() 2304 @skipIfNoNetwork() 2305 def test_real_git_lfs_repo_skips(self): 2306 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=0") 2307 f = self.get_real_git_lfs_file() 2308 # This is the actual non-smudged placeholder file on the repo if git-lfs does not run 2309 lfs_file = ( 2310 'version https://git-lfs.github.com/spec/v1\n' 2311 'oid sha256:34be66b1a39a1955b46a12588df9d5f6fc1da790e05cf01f3c7422f4bbbdc26b\n' 2312 'size 11423554\n' 2313 ) 2314 2315 with open(f) as fh: 2316 self.assertEqual(lfs_file, fh.read()) 2317 2318 @skipIfNoGitLFS() 2319 def test_lfs_enabled(self): 2320 import shutil 2321 2322 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir 2323 self.d.setVar('SRC_URI', uri) 2324 2325 # With git-lfs installed, test that we can fetch and unpack 2326 fetcher, ud = self.fetch() 2327 shutil.rmtree(self.gitdir, ignore_errors=True) 2328 fetcher.unpack(self.d.getVar('WORKDIR')) 2329 2330 @skipIfNoGitLFS() 2331 def test_lfs_disabled(self): 2332 import shutil 2333 2334 uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir 2335 self.d.setVar('SRC_URI', uri) 2336 2337 # Verify that the fetcher can survive even if the source 2338 # repository has Git LFS usage configured. 2339 fetcher, ud = self.fetch() 2340 fetcher.unpack(self.d.getVar('WORKDIR')) 2341 2342 def test_lfs_enabled_not_installed(self): 2343 import shutil 2344 2345 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir 2346 self.d.setVar('SRC_URI', uri) 2347 2348 # Careful: suppress initial attempt at downloading 2349 fetcher, ud = self.fetch(uri=None, download=False) 2350 2351 # Artificially assert that git-lfs is not installed, so 2352 # we can verify a failure to unpack in it's absence. 2353 old_find_git_lfs = ud.method._find_git_lfs 2354 try: 2355 # If git-lfs cannot be found, the unpack should throw an error 2356 with self.assertRaises(bb.fetch2.FetchError): 2357 fetcher.download() 2358 ud.method._find_git_lfs = lambda d: False 2359 shutil.rmtree(self.gitdir, ignore_errors=True) 2360 fetcher.unpack(self.d.getVar('WORKDIR')) 2361 finally: 2362 ud.method._find_git_lfs = old_find_git_lfs 2363 2364 def test_lfs_disabled_not_installed(self): 2365 import shutil 2366 2367 uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir 2368 self.d.setVar('SRC_URI', uri) 2369 2370 # Careful: suppress initial attempt at downloading 2371 fetcher, ud = self.fetch(uri=None, download=False) 2372 2373 # Artificially assert that git-lfs is not installed, so 2374 # we can verify a failure to unpack in it's absence. 2375 old_find_git_lfs = ud.method._find_git_lfs 2376 try: 2377 # Even if git-lfs cannot be found, the unpack should be successful 2378 fetcher.download() 2379 ud.method._find_git_lfs = lambda d: False 2380 shutil.rmtree(self.gitdir, ignore_errors=True) 2381 fetcher.unpack(self.d.getVar('WORKDIR')) 2382 finally: 2383 ud.method._find_git_lfs = old_find_git_lfs 2384 2385class GitURLWithSpacesTest(FetcherTest): 2386 test_git_urls = { 2387 "git://tfs-example.org:22/tfs/example%20path/example.git;branch=master" : { 2388 'url': 'git://tfs-example.org:22/tfs/example%20path/example.git;branch=master', 2389 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example.git', 2390 'path': '/tfs/example path/example.git' 2391 }, 2392 "git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master" : { 2393 'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master', 2394 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example_repo.git', 2395 'path': '/tfs/example path/example repo.git' 2396 } 2397 } 2398 2399 def test_urls(self): 2400 2401 # Set fake SRCREV to stop git fetcher from trying to contact non-existent git repo 2402 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 2403 2404 for test_git_url, ref in self.test_git_urls.items(): 2405 2406 fetcher = bb.fetch.Fetch([test_git_url], self.d) 2407 ud = fetcher.ud[fetcher.urls[0]] 2408 2409 self.assertEqual(ud.url, ref['url']) 2410 self.assertEqual(ud.path, ref['path']) 2411 self.assertEqual(ud.localfile, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2412 self.assertEqual(ud.localpath, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2413 self.assertEqual(ud.lockfile, os.path.join(self.dldir, "git2", ref['gitsrcname'] + '.lock')) 2414 self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2415 self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz')) 2416 2417class CrateTest(FetcherTest): 2418 @skipIfNoNetwork() 2419 def test_crate_url(self): 2420 2421 uri = "crate://crates.io/glob/0.2.11" 2422 self.d.setVar('SRC_URI', uri) 2423 2424 uris = self.d.getVar('SRC_URI').split() 2425 d = self.d 2426 2427 fetcher = bb.fetch2.Fetch(uris, self.d) 2428 ud = fetcher.ud[fetcher.urls[0]] 2429 2430 self.assertIn("name", ud.parm) 2431 self.assertEqual(ud.parm["name"], "glob-0.2.11") 2432 self.assertIn("downloadfilename", ud.parm) 2433 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") 2434 2435 fetcher.download() 2436 fetcher.unpack(self.tempdir) 2437 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2438 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done']) 2439 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) 2440 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) 2441 2442 @skipIfNoNetwork() 2443 def test_crate_url_matching_recipe(self): 2444 2445 self.d.setVar('BP', 'glob-0.2.11') 2446 2447 uri = "crate://crates.io/glob/0.2.11" 2448 self.d.setVar('SRC_URI', uri) 2449 2450 uris = self.d.getVar('SRC_URI').split() 2451 d = self.d 2452 2453 fetcher = bb.fetch2.Fetch(uris, self.d) 2454 ud = fetcher.ud[fetcher.urls[0]] 2455 2456 self.assertIn("name", ud.parm) 2457 self.assertEqual(ud.parm["name"], "glob-0.2.11") 2458 self.assertIn("downloadfilename", ud.parm) 2459 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") 2460 2461 fetcher.download() 2462 fetcher.unpack(self.tempdir) 2463 self.assertEqual(sorted(os.listdir(self.tempdir)), ['download', 'glob-0.2.11', 'unpacked']) 2464 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done']) 2465 self.assertTrue(os.path.exists(self.tempdir + "/glob-0.2.11/src/lib.rs")) 2466 2467 @skipIfNoNetwork() 2468 def test_crate_url_params(self): 2469 2470 uri = "crate://crates.io/aho-corasick/0.7.20;name=aho-corasick-renamed" 2471 self.d.setVar('SRC_URI', uri) 2472 2473 uris = self.d.getVar('SRC_URI').split() 2474 d = self.d 2475 2476 fetcher = bb.fetch2.Fetch(uris, self.d) 2477 ud = fetcher.ud[fetcher.urls[0]] 2478 2479 self.assertIn("name", ud.parm) 2480 self.assertEqual(ud.parm["name"], "aho-corasick-renamed") 2481 self.assertIn("downloadfilename", ud.parm) 2482 self.assertEqual(ud.parm["downloadfilename"], "aho-corasick-0.7.20.crate") 2483 2484 fetcher.download() 2485 fetcher.unpack(self.tempdir) 2486 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2487 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['aho-corasick-0.7.20.crate', 'aho-corasick-0.7.20.crate.done']) 2488 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/.cargo-checksum.json")) 2489 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/src/lib.rs")) 2490 2491 @skipIfNoNetwork() 2492 def test_crate_url_multi(self): 2493 2494 uri = "crate://crates.io/glob/0.2.11 crate://crates.io/time/0.1.35" 2495 self.d.setVar('SRC_URI', uri) 2496 2497 uris = self.d.getVar('SRC_URI').split() 2498 d = self.d 2499 2500 fetcher = bb.fetch2.Fetch(uris, self.d) 2501 ud = fetcher.ud[fetcher.urls[0]] 2502 2503 self.assertIn("name", ud.parm) 2504 self.assertEqual(ud.parm["name"], "glob-0.2.11") 2505 self.assertIn("downloadfilename", ud.parm) 2506 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") 2507 2508 ud = fetcher.ud[fetcher.urls[1]] 2509 self.assertIn("name", ud.parm) 2510 self.assertEqual(ud.parm["name"], "time-0.1.35") 2511 self.assertIn("downloadfilename", ud.parm) 2512 self.assertEqual(ud.parm["downloadfilename"], "time-0.1.35.crate") 2513 2514 fetcher.download() 2515 fetcher.unpack(self.tempdir) 2516 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2517 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done', 'time-0.1.35.crate', 'time-0.1.35.crate.done']) 2518 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) 2519 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) 2520 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/.cargo-checksum.json")) 2521 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/src/lib.rs")) 2522 2523 @skipIfNoNetwork() 2524 def test_crate_incorrect_cksum(self): 2525 uri = "crate://crates.io/aho-corasick/0.7.20" 2526 self.d.setVar('SRC_URI', uri) 2527 self.d.setVarFlag("SRC_URI", "aho-corasick-0.7.20.sha256sum", hashlib.sha256("Invalid".encode("utf-8")).hexdigest()) 2528 2529 uris = self.d.getVar('SRC_URI').split() 2530 2531 fetcher = bb.fetch2.Fetch(uris, self.d) 2532 with self.assertRaisesRegex(bb.fetch2.FetchError, "Fetcher failure for URL"): 2533 fetcher.download() 2534 2535class NPMTest(FetcherTest): 2536 def skipIfNoNpm(): 2537 import shutil 2538 if not shutil.which('npm'): 2539 return unittest.skip('npm not installed') 2540 return lambda f: f 2541 2542 @skipIfNoNpm() 2543 @skipIfNoNetwork() 2544 def test_npm(self): 2545 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2546 fetcher = bb.fetch.Fetch([url], self.d) 2547 ud = fetcher.ud[fetcher.urls[0]] 2548 fetcher.download() 2549 self.assertTrue(os.path.exists(ud.localpath)) 2550 self.assertTrue(os.path.exists(ud.localpath + '.done')) 2551 self.assertTrue(os.path.exists(ud.resolvefile)) 2552 fetcher.unpack(self.unpackdir) 2553 unpackdir = os.path.join(self.unpackdir, 'npm') 2554 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2555 2556 @skipIfNoNpm() 2557 @skipIfNoNetwork() 2558 def test_npm_bad_checksum(self): 2559 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2560 # Fetch once to get a tarball 2561 fetcher = bb.fetch.Fetch([url], self.d) 2562 ud = fetcher.ud[fetcher.urls[0]] 2563 fetcher.download() 2564 self.assertTrue(os.path.exists(ud.localpath)) 2565 # Modify the tarball 2566 bad = b'bad checksum' 2567 with open(ud.localpath, 'wb') as f: 2568 f.write(bad) 2569 # Verify that the tarball is fetched again 2570 fetcher.download() 2571 badsum = hashlib.sha512(bad).hexdigest() 2572 self.assertTrue(os.path.exists(ud.localpath + '_bad-checksum_' + badsum)) 2573 self.assertTrue(os.path.exists(ud.localpath)) 2574 2575 @skipIfNoNpm() 2576 @skipIfNoNetwork() 2577 def test_npm_premirrors(self): 2578 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2579 # Fetch once to get a tarball 2580 fetcher = bb.fetch.Fetch([url], self.d) 2581 ud = fetcher.ud[fetcher.urls[0]] 2582 fetcher.download() 2583 self.assertTrue(os.path.exists(ud.localpath)) 2584 2585 # Setup the mirror by renaming the download directory 2586 mirrordir = os.path.join(self.tempdir, 'mirror') 2587 bb.utils.rename(self.dldir, mirrordir) 2588 os.mkdir(self.dldir) 2589 2590 # Configure the premirror to be used 2591 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/npm2' % mirrordir) 2592 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2593 2594 # Fetch again 2595 self.assertFalse(os.path.exists(ud.localpath)) 2596 # The npm fetcher doesn't handle that the .resolved file disappears 2597 # while the fetcher object exists, which it does when we rename the 2598 # download directory to "mirror" above. Thus we need a new fetcher to go 2599 # with the now empty download directory. 2600 fetcher = bb.fetch.Fetch([url], self.d) 2601 ud = fetcher.ud[fetcher.urls[0]] 2602 fetcher.download() 2603 self.assertTrue(os.path.exists(ud.localpath)) 2604 2605 @skipIfNoNpm() 2606 @skipIfNoNetwork() 2607 def test_npm_premirrors_with_specified_filename(self): 2608 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2609 # Fetch once to get a tarball 2610 fetcher = bb.fetch.Fetch([url], self.d) 2611 ud = fetcher.ud[fetcher.urls[0]] 2612 fetcher.download() 2613 self.assertTrue(os.path.exists(ud.localpath)) 2614 # Setup the mirror 2615 mirrordir = os.path.join(self.tempdir, 'mirror') 2616 bb.utils.mkdirhier(mirrordir) 2617 mirrorfilename = os.path.join(mirrordir, os.path.basename(ud.localpath)) 2618 os.replace(ud.localpath, mirrorfilename) 2619 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s' % mirrorfilename) 2620 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2621 # Fetch again 2622 self.assertFalse(os.path.exists(ud.localpath)) 2623 fetcher.download() 2624 self.assertTrue(os.path.exists(ud.localpath)) 2625 2626 @skipIfNoNpm() 2627 @skipIfNoNetwork() 2628 def test_npm_mirrors(self): 2629 # Fetch once to get a tarball 2630 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2631 fetcher = bb.fetch.Fetch([url], self.d) 2632 ud = fetcher.ud[fetcher.urls[0]] 2633 fetcher.download() 2634 self.assertTrue(os.path.exists(ud.localpath)) 2635 # Setup the mirror 2636 mirrordir = os.path.join(self.tempdir, 'mirror') 2637 bb.utils.mkdirhier(mirrordir) 2638 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 2639 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 2640 # Update the resolved url to an invalid url 2641 with open(ud.resolvefile, 'r') as f: 2642 url = f.read() 2643 uri = URI(url) 2644 uri.path = '/invalid' 2645 with open(ud.resolvefile, 'w') as f: 2646 f.write(str(uri)) 2647 # Fetch again 2648 self.assertFalse(os.path.exists(ud.localpath)) 2649 fetcher.download() 2650 self.assertTrue(os.path.exists(ud.localpath)) 2651 2652 @skipIfNoNpm() 2653 @skipIfNoNetwork() 2654 def test_npm_destsuffix_downloadfilename(self): 2655 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz' 2656 fetcher = bb.fetch.Fetch([url], self.d) 2657 fetcher.download() 2658 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'foo-bar.tgz'))) 2659 fetcher.unpack(self.unpackdir) 2660 unpackdir = os.path.join(self.unpackdir, 'foo', 'bar') 2661 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2662 2663 def test_npm_no_network_no_tarball(self): 2664 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2665 self.d.setVar('BB_NO_NETWORK', '1') 2666 fetcher = bb.fetch.Fetch([url], self.d) 2667 with self.assertRaises(bb.fetch2.NetworkAccess): 2668 fetcher.download() 2669 2670 @skipIfNoNpm() 2671 @skipIfNoNetwork() 2672 def test_npm_no_network_with_tarball(self): 2673 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2674 # Fetch once to get a tarball 2675 fetcher = bb.fetch.Fetch([url], self.d) 2676 fetcher.download() 2677 # Disable network access 2678 self.d.setVar('BB_NO_NETWORK', '1') 2679 # Fetch again 2680 fetcher.download() 2681 fetcher.unpack(self.unpackdir) 2682 unpackdir = os.path.join(self.unpackdir, 'npm') 2683 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2684 2685 @skipIfNoNpm() 2686 @skipIfNoNetwork() 2687 def test_npm_registry_alternate(self): 2688 url = 'npm://skimdb.npmjs.com;package=@savoirfairelinux/node-server-example;version=1.0.0' 2689 fetcher = bb.fetch.Fetch([url], self.d) 2690 fetcher.download() 2691 fetcher.unpack(self.unpackdir) 2692 unpackdir = os.path.join(self.unpackdir, 'npm') 2693 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2694 2695 @skipIfNoNpm() 2696 @skipIfNoNetwork() 2697 def test_npm_version_latest(self): 2698 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=latest' 2699 fetcher = bb.fetch.Fetch([url], self.d) 2700 fetcher.download() 2701 fetcher.unpack(self.unpackdir) 2702 unpackdir = os.path.join(self.unpackdir, 'npm') 2703 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2704 2705 @skipIfNoNpm() 2706 @skipIfNoNetwork() 2707 def test_npm_registry_invalid(self): 2708 url = 'npm://registry.invalid.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2709 fetcher = bb.fetch.Fetch([url], self.d) 2710 with self.assertRaises(bb.fetch2.FetchError): 2711 fetcher.download() 2712 2713 @skipIfNoNpm() 2714 @skipIfNoNetwork() 2715 def test_npm_package_invalid(self): 2716 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/invalid;version=1.0.0' 2717 fetcher = bb.fetch.Fetch([url], self.d) 2718 with self.assertRaises(bb.fetch2.FetchError): 2719 fetcher.download() 2720 2721 @skipIfNoNpm() 2722 @skipIfNoNetwork() 2723 def test_npm_version_invalid(self): 2724 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=invalid' 2725 with self.assertRaises(bb.fetch2.ParameterError): 2726 fetcher = bb.fetch.Fetch([url], self.d) 2727 2728 @skipIfNoNpm() 2729 @skipIfNoNetwork() 2730 def test_npm_registry_none(self): 2731 url = 'npm://;package=@savoirfairelinux/node-server-example;version=1.0.0' 2732 with self.assertRaises(bb.fetch2.MalformedUrl): 2733 fetcher = bb.fetch.Fetch([url], self.d) 2734 2735 @skipIfNoNpm() 2736 @skipIfNoNetwork() 2737 def test_npm_package_none(self): 2738 url = 'npm://registry.npmjs.org;version=1.0.0' 2739 with self.assertRaises(bb.fetch2.MissingParameterError): 2740 fetcher = bb.fetch.Fetch([url], self.d) 2741 2742 @skipIfNoNpm() 2743 @skipIfNoNetwork() 2744 def test_npm_version_none(self): 2745 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example' 2746 with self.assertRaises(bb.fetch2.MissingParameterError): 2747 fetcher = bb.fetch.Fetch([url], self.d) 2748 2749 def create_shrinkwrap_file(self, data): 2750 import json 2751 datadir = os.path.join(self.tempdir, 'data') 2752 swfile = os.path.join(datadir, 'npm-shrinkwrap.json') 2753 bb.utils.mkdirhier(datadir) 2754 with open(swfile, 'w') as f: 2755 json.dump(data, f) 2756 # Also configure the S directory 2757 self.sdir = os.path.join(self.unpackdir, 'S') 2758 self.d.setVar('S', self.sdir) 2759 return swfile 2760 2761 @skipIfNoNpm() 2762 @skipIfNoNetwork() 2763 def test_npmsw(self): 2764 swfile = self.create_shrinkwrap_file({ 2765 'dependencies': { 2766 'array-flatten': { 2767 'version': '1.1.1', 2768 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2769 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=', 2770 'dependencies': { 2771 'content-type': { 2772 'version': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', 2773 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', 2774 'dependencies': { 2775 'cookie': { 2776 'version': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09', 2777 'from': 'git+https://github.com/jshttp/cookie.git' 2778 } 2779 } 2780 } 2781 } 2782 } 2783 } 2784 }) 2785 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2786 fetcher.download() 2787 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2788 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2789 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) 2790 fetcher.unpack(self.unpackdir) 2791 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'npm-shrinkwrap.json'))) 2792 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json'))) 2793 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'package.json'))) 2794 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'node_modules', 'cookie', 'package.json'))) 2795 2796 @skipIfNoNpm() 2797 @skipIfNoNetwork() 2798 def test_npmsw_git(self): 2799 swfile = self.create_shrinkwrap_file({ 2800 'dependencies': { 2801 'cookie': { 2802 'version': 'github:jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09', 2803 'from': 'github:jshttp/cookie.git' 2804 } 2805 } 2806 }) 2807 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2808 fetcher.download() 2809 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) 2810 2811 swfile = self.create_shrinkwrap_file({ 2812 'dependencies': { 2813 'cookie': { 2814 'version': 'jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09', 2815 'from': 'jshttp/cookie.git' 2816 } 2817 } 2818 }) 2819 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2820 fetcher.download() 2821 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) 2822 2823 swfile = self.create_shrinkwrap_file({ 2824 'dependencies': { 2825 'nodejs': { 2826 'version': 'gitlab:gitlab-examples/nodejs.git#892a1f16725e56cc3a2cb0d677be42935c8fc262', 2827 'from': 'gitlab:gitlab-examples/nodejs' 2828 } 2829 } 2830 }) 2831 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2832 fetcher.download() 2833 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'gitlab.com.gitlab-examples.nodejs.git'))) 2834 2835 @skipIfNoNpm() 2836 @skipIfNoNetwork() 2837 def test_npmsw_dev(self): 2838 swfile = self.create_shrinkwrap_file({ 2839 'dependencies': { 2840 'array-flatten': { 2841 'version': '1.1.1', 2842 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2843 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2844 }, 2845 'content-type': { 2846 'version': '1.0.4', 2847 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', 2848 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', 2849 'dev': True 2850 } 2851 } 2852 }) 2853 # Fetch with dev disabled 2854 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2855 fetcher.download() 2856 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2857 self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2858 # Fetch with dev enabled 2859 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';dev=1'], self.d) 2860 fetcher.download() 2861 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2862 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2863 2864 @skipIfNoNpm() 2865 @skipIfNoNetwork() 2866 def test_npmsw_destsuffix(self): 2867 swfile = self.create_shrinkwrap_file({ 2868 'dependencies': { 2869 'array-flatten': { 2870 'version': '1.1.1', 2871 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2872 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2873 } 2874 } 2875 }) 2876 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';destsuffix=foo/bar'], self.d) 2877 fetcher.download() 2878 fetcher.unpack(self.unpackdir) 2879 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'foo', 'bar', 'node_modules', 'array-flatten', 'package.json'))) 2880 2881 def test_npmsw_no_network_no_tarball(self): 2882 swfile = self.create_shrinkwrap_file({ 2883 'dependencies': { 2884 'array-flatten': { 2885 'version': '1.1.1', 2886 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2887 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2888 } 2889 } 2890 }) 2891 self.d.setVar('BB_NO_NETWORK', '1') 2892 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2893 with self.assertRaises(bb.fetch2.NetworkAccess): 2894 fetcher.download() 2895 2896 @skipIfNoNpm() 2897 @skipIfNoNetwork() 2898 def test_npmsw_no_network_with_tarball(self): 2899 # Fetch once to get a tarball 2900 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 2901 fetcher.download() 2902 # Disable network access 2903 self.d.setVar('BB_NO_NETWORK', '1') 2904 # Fetch again 2905 swfile = self.create_shrinkwrap_file({ 2906 'dependencies': { 2907 'array-flatten': { 2908 'version': '1.1.1', 2909 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2910 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2911 } 2912 } 2913 }) 2914 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2915 fetcher.download() 2916 fetcher.unpack(self.unpackdir) 2917 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json'))) 2918 2919 @skipIfNoNpm() 2920 @skipIfNoNetwork() 2921 def test_npmsw_npm_reusability(self): 2922 # Fetch once with npmsw 2923 swfile = self.create_shrinkwrap_file({ 2924 'dependencies': { 2925 'array-flatten': { 2926 'version': '1.1.1', 2927 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2928 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2929 } 2930 } 2931 }) 2932 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2933 fetcher.download() 2934 # Disable network access 2935 self.d.setVar('BB_NO_NETWORK', '1') 2936 # Fetch again with npm 2937 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 2938 fetcher.download() 2939 fetcher.unpack(self.unpackdir) 2940 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm', 'package.json'))) 2941 2942 @skipIfNoNpm() 2943 @skipIfNoNetwork() 2944 def test_npmsw_bad_checksum(self): 2945 # Try to fetch with bad checksum 2946 swfile = self.create_shrinkwrap_file({ 2947 'dependencies': { 2948 'array-flatten': { 2949 'version': '1.1.1', 2950 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2951 'integrity': 'sha1-gfNEp2hqgLTFKT6P3AsBYMgsBqg=' 2952 } 2953 } 2954 }) 2955 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2956 with self.assertRaises(bb.fetch2.FetchError): 2957 fetcher.download() 2958 # Fetch correctly to get a tarball 2959 swfile = self.create_shrinkwrap_file({ 2960 'dependencies': { 2961 'array-flatten': { 2962 'version': '1.1.1', 2963 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2964 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2965 } 2966 } 2967 }) 2968 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2969 fetcher.download() 2970 localpath = os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz') 2971 self.assertTrue(os.path.exists(localpath)) 2972 # Modify the tarball 2973 bad = b'bad checksum' 2974 with open(localpath, 'wb') as f: 2975 f.write(bad) 2976 # Verify that the tarball is fetched again 2977 fetcher.download() 2978 badsum = hashlib.sha1(bad).hexdigest() 2979 self.assertTrue(os.path.exists(localpath + '_bad-checksum_' + badsum)) 2980 self.assertTrue(os.path.exists(localpath)) 2981 2982 @skipIfNoNpm() 2983 @skipIfNoNetwork() 2984 def test_npmsw_premirrors(self): 2985 # Fetch once to get a tarball 2986 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 2987 ud = fetcher.ud[fetcher.urls[0]] 2988 fetcher.download() 2989 self.assertTrue(os.path.exists(ud.localpath)) 2990 # Setup the mirror 2991 mirrordir = os.path.join(self.tempdir, 'mirror') 2992 bb.utils.mkdirhier(mirrordir) 2993 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 2994 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 2995 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2996 # Fetch again 2997 self.assertFalse(os.path.exists(ud.localpath)) 2998 swfile = self.create_shrinkwrap_file({ 2999 'dependencies': { 3000 'array-flatten': { 3001 'version': '1.1.1', 3002 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3003 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3004 } 3005 } 3006 }) 3007 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3008 fetcher.download() 3009 self.assertTrue(os.path.exists(ud.localpath)) 3010 3011 @skipIfNoNpm() 3012 @skipIfNoNetwork() 3013 def test_npmsw_mirrors(self): 3014 # Fetch once to get a tarball 3015 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 3016 ud = fetcher.ud[fetcher.urls[0]] 3017 fetcher.download() 3018 self.assertTrue(os.path.exists(ud.localpath)) 3019 # Setup the mirror 3020 mirrordir = os.path.join(self.tempdir, 'mirror') 3021 bb.utils.mkdirhier(mirrordir) 3022 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 3023 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 3024 # Fetch again with invalid url 3025 self.assertFalse(os.path.exists(ud.localpath)) 3026 swfile = self.create_shrinkwrap_file({ 3027 'dependencies': { 3028 'array-flatten': { 3029 'version': '1.1.1', 3030 'resolved': 'https://invalid', 3031 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3032 } 3033 } 3034 }) 3035 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3036 fetcher.download() 3037 self.assertTrue(os.path.exists(ud.localpath)) 3038 3039class GitSharedTest(FetcherTest): 3040 def setUp(self): 3041 super(GitSharedTest, self).setUp() 3042 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 3043 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 3044 self.d.setVar("__BBSRCREV_SEEN", "1") 3045 3046 @skipIfNoNetwork() 3047 def test_shared_unpack(self): 3048 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3049 3050 fetcher.download() 3051 fetcher.unpack(self.unpackdir) 3052 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 3053 self.assertTrue(os.path.exists(alt)) 3054 3055 @skipIfNoNetwork() 3056 def test_noshared_unpack(self): 3057 self.d.setVar('BB_GIT_NOSHARED', '1') 3058 self.unpackdir += '_noshared' 3059 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3060 3061 fetcher.download() 3062 fetcher.unpack(self.unpackdir) 3063 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 3064 self.assertFalse(os.path.exists(alt)) 3065 3066 3067class FetchPremirroronlyLocalTest(FetcherTest): 3068 3069 def setUp(self): 3070 super(FetchPremirroronlyLocalTest, self).setUp() 3071 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3072 os.mkdir(self.mirrordir) 3073 self.reponame = "bitbake" 3074 self.gitdir = os.path.join(self.tempdir, "git", self.reponame) 3075 self.recipe_url = "git://git.fake.repo/bitbake;branch=master;protocol=https" 3076 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3077 self.d.setVar("BB_NO_NETWORK", "1") 3078 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3079 self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz" 3080 self.mirrorfile = os.path.join(self.mirrordir, self.mirrorname) 3081 self.testfilename = "bitbake-fetch.test" 3082 3083 def make_git_repo(self): 3084 recipeurl = "git:/git.fake.repo/bitbake" 3085 os.makedirs(self.gitdir) 3086 self.git_init(cwd=self.gitdir) 3087 for i in range(0): 3088 self.git_new_commit() 3089 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3090 3091 def git_new_commit(self): 3092 import random 3093 os.unlink(os.path.join(self.mirrordir, self.mirrorname)) 3094 branch = self.git("branch --show-current", self.gitdir).split() 3095 with open(os.path.join(self.gitdir, self.testfilename), "w") as testfile: 3096 testfile.write("File {} from branch {}; Useless random data {}".format(self.testfilename, branch, random.random())) 3097 self.git("add {}".format(self.testfilename), self.gitdir) 3098 self.git("commit -a -m \"This random commit {} in branch {}. I'm useless.\"".format(random.random(), branch), self.gitdir) 3099 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3100 return self.git("rev-parse HEAD", self.gitdir).strip() 3101 3102 def git_new_branch(self, name): 3103 self.git_new_commit() 3104 head = self.git("rev-parse HEAD", self.gitdir).strip() 3105 self.git("checkout -b {}".format(name), self.gitdir) 3106 newrev = self.git_new_commit() 3107 self.git("checkout {}".format(head), self.gitdir) 3108 return newrev 3109 3110 def test_mirror_multiple_fetches(self): 3111 self.make_git_repo() 3112 self.d.setVar("SRCREV", self.git_new_commit()) 3113 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3114 fetcher.download() 3115 fetcher.unpack(self.unpackdir) 3116 ## New commit in premirror. it's not in the download_dir 3117 self.d.setVar("SRCREV", self.git_new_commit()) 3118 fetcher2 = bb.fetch.Fetch([self.recipe_url], self.d) 3119 fetcher2.download() 3120 fetcher2.unpack(self.unpackdir) 3121 ## New commit in premirror. it's not in the download_dir 3122 self.d.setVar("SRCREV", self.git_new_commit()) 3123 fetcher3 = bb.fetch.Fetch([self.recipe_url], self.d) 3124 fetcher3.download() 3125 fetcher3.unpack(self.unpackdir) 3126 3127 3128 def test_mirror_commit_nonexistent(self): 3129 self.make_git_repo() 3130 self.d.setVar("SRCREV", "0"*40) 3131 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3132 with self.assertRaises(bb.fetch2.NetworkAccess): 3133 fetcher.download() 3134 3135 def test_mirror_commit_exists(self): 3136 self.make_git_repo() 3137 self.d.setVar("SRCREV", self.git_new_commit()) 3138 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3139 fetcher.download() 3140 fetcher.unpack(self.unpackdir) 3141 3142 def test_mirror_tarball_nonexistent(self): 3143 self.d.setVar("SRCREV", "0"*40) 3144 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3145 with self.assertRaises(bb.fetch2.NetworkAccess): 3146 fetcher.download() 3147 3148 def test_mirror_tarball_multiple_branches(self): 3149 """ 3150 test if PREMIRRORS can handle multiple name/branches correctly 3151 both branches have required revisions 3152 """ 3153 self.make_git_repo() 3154 branch1rev = self.git_new_branch("testbranch1") 3155 branch2rev = self.git_new_branch("testbranch2") 3156 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1,testbranch2;protocol=https;name=branch1,branch2" 3157 self.d.setVar("SRCREV_branch1", branch1rev) 3158 self.d.setVar("SRCREV_branch2", branch2rev) 3159 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3160 self.assertTrue(os.path.exists(self.mirrorfile), "Mirror file doesn't exist") 3161 fetcher.download() 3162 fetcher.unpack(os.path.join(self.tempdir, "unpacked")) 3163 unpacked = os.path.join(self.tempdir, "unpacked", "git", self.testfilename) 3164 self.assertTrue(os.path.exists(unpacked), "Repo has not been unpackaged properly!") 3165 with open(unpacked, 'r') as f: 3166 content = f.read() 3167 ## We expect to see testbranch1 in the file, not master, not testbranch2 3168 self.assertTrue(content.find("testbranch1") != -1, "Wrong branch has been checked out!") 3169 3170 def test_mirror_tarball_multiple_branches_nobranch(self): 3171 """ 3172 test if PREMIRRORS can handle multiple name/branches correctly 3173 Unbalanced name/branches raises ParameterError 3174 """ 3175 self.make_git_repo() 3176 branch1rev = self.git_new_branch("testbranch1") 3177 branch2rev = self.git_new_branch("testbranch2") 3178 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1;protocol=https;name=branch1,branch2" 3179 self.d.setVar("SRCREV_branch1", branch1rev) 3180 self.d.setVar("SRCREV_branch2", branch2rev) 3181 with self.assertRaises(bb.fetch2.ParameterError): 3182 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3183 3184 def test_mirror_tarball_multiple_branches_norev(self): 3185 """ 3186 test if PREMIRRORS can handle multiple name/branches correctly 3187 one of the branches specifies non existing SRCREV 3188 """ 3189 self.make_git_repo() 3190 branch1rev = self.git_new_branch("testbranch1") 3191 branch2rev = self.git_new_branch("testbranch2") 3192 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1,testbranch2;protocol=https;name=branch1,branch2" 3193 self.d.setVar("SRCREV_branch1", branch1rev) 3194 self.d.setVar("SRCREV_branch2", "0"*40) 3195 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3196 self.assertTrue(os.path.exists(self.mirrorfile), "Mirror file doesn't exist") 3197 with self.assertRaises(bb.fetch2.NetworkAccess): 3198 fetcher.download() 3199 3200 3201class FetchPremirroronlyNetworkTest(FetcherTest): 3202 3203 def setUp(self): 3204 super(FetchPremirroronlyNetworkTest, self).setUp() 3205 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3206 os.mkdir(self.mirrordir) 3207 self.reponame = "fstests" 3208 self.clonedir = os.path.join(self.tempdir, "git") 3209 self.gitdir = os.path.join(self.tempdir, "git", "{}.git".format(self.reponame)) 3210 self.recipe_url = "git://git.yoctoproject.org/fstests;protocol=https" 3211 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3212 self.d.setVar("BB_NO_NETWORK", "0") 3213 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3214 3215 def make_git_repo(self): 3216 import shutil 3217 self.mirrorname = "git2_git.yoctoproject.org.fstests.tar.gz" 3218 os.makedirs(self.clonedir) 3219 self.git("clone --bare --shallow-since=\"01.01.2013\" {}".format(self.recipe_url), self.clonedir) 3220 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3221 shutil.rmtree(self.clonedir) 3222 3223 @skipIfNoNetwork() 3224 def test_mirror_tarball_updated(self): 3225 self.make_git_repo() 3226 ## Upstream commit is in the mirror 3227 self.d.setVar("SRCREV", "49d65d53c2bf558ae6e9185af0f3af7b79d255ec") 3228 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3229 fetcher.download() 3230 3231 @skipIfNoNetwork() 3232 def test_mirror_tarball_outdated(self): 3233 self.make_git_repo() 3234 ## Upstream commit not in the mirror 3235 self.d.setVar("SRCREV", "15413486df1f5a5b5af699b6f3ba5f0984e52a9f") 3236 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3237 with self.assertRaises(bb.fetch2.NetworkAccess): 3238 fetcher.download() 3239 3240class FetchPremirroronlyMercurialTest(FetcherTest): 3241 """ Test for premirrors with mercurial repos 3242 the test covers also basic hg:// clone (see fetch_and_create_tarball 3243 """ 3244 def skipIfNoHg(): 3245 import shutil 3246 if not shutil.which('hg'): 3247 return unittest.skip('Mercurial not installed') 3248 return lambda f: f 3249 3250 def setUp(self): 3251 super(FetchPremirroronlyMercurialTest, self).setUp() 3252 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3253 os.mkdir(self.mirrordir) 3254 self.reponame = "libgnt" 3255 self.clonedir = os.path.join(self.tempdir, "hg") 3256 self.recipe_url = "hg://keep.imfreedom.org/libgnt;module=libgnt" 3257 self.d.setVar("SRCREV", "53e8b422faaf") 3258 self.mirrorname = "hg_libgnt_keep.imfreedom.org_.libgnt.tar.gz" 3259 3260 def fetch_and_create_tarball(self): 3261 """ 3262 Ask bitbake to download repo and prepare mirror tarball for us 3263 """ 3264 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") 3265 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3266 fetcher.download() 3267 mirrorfile = os.path.join(self.d.getVar("DL_DIR"), self.mirrorname) 3268 self.assertTrue(os.path.exists(mirrorfile), "Mirror tarball {} has not been created".format(mirrorfile)) 3269 ## moving tarball to mirror directory 3270 os.rename(mirrorfile, os.path.join(self.mirrordir, self.mirrorname)) 3271 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "0") 3272 3273 3274 @skipIfNoNetwork() 3275 @skipIfNoHg() 3276 def test_premirror_mercurial(self): 3277 self.fetch_and_create_tarball() 3278 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3279 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3280 self.d.setVar("BB_NO_NETWORK", "1") 3281 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3282 fetcher.download() 3283 3284class FetchPremirroronlyBrokenTarball(FetcherTest): 3285 3286 def setUp(self): 3287 super(FetchPremirroronlyBrokenTarball, self).setUp() 3288 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3289 os.mkdir(self.mirrordir) 3290 self.reponame = "bitbake" 3291 self.gitdir = os.path.join(self.tempdir, "git", self.reponame) 3292 self.recipe_url = "git://git.fake.repo/bitbake;protocol=https" 3293 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3294 self.d.setVar("BB_NO_NETWORK", "1") 3295 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3296 self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz" 3297 with open(os.path.join(self.mirrordir, self.mirrorname), 'w') as targz: 3298 targz.write("This is not tar.gz file!") 3299 3300 def test_mirror_broken_download(self): 3301 import sys 3302 self.d.setVar("SRCREV", "0"*40) 3303 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3304 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs() as logs: 3305 fetcher.download() 3306 output = "".join(logs.output) 3307 self.assertFalse(" not a git repository (or any parent up to mount point /)" in output) 3308