1# 2# BitBake Tests for the Fetcher (fetch2/) 3# 4# Copyright (C) 2012 Richard Purdie 5# 6# SPDX-License-Identifier: GPL-2.0-only 7# 8 9import contextlib 10import unittest 11import hashlib 12import tempfile 13import collections 14import os 15import signal 16import tarfile 17from bb.fetch2 import URI 18from bb.fetch2 import FetchMethod 19import bb 20from bb.tests.support.httpserver import HTTPService 21 22def skipIfNoNetwork(): 23 if os.environ.get("BB_SKIP_NETTESTS") == "yes": 24 return unittest.skip("network test") 25 return lambda f: f 26 27class TestTimeout(Exception): 28 # Indicate to pytest that this is not a test suite 29 __test__ = False 30 31class Timeout(): 32 33 def __init__(self, seconds): 34 self.seconds = seconds 35 36 def handle_timeout(self, signum, frame): 37 raise TestTimeout("Test failed: timeout reached") 38 39 def __enter__(self): 40 signal.signal(signal.SIGALRM, self.handle_timeout) 41 signal.alarm(self.seconds) 42 43 def __exit__(self, exc_type, exc_val, exc_tb): 44 signal.alarm(0) 45 46class URITest(unittest.TestCase): 47 test_uris = { 48 "http://www.google.com/index.html" : { 49 'uri': 'http://www.google.com/index.html', 50 'scheme': 'http', 51 'hostname': 'www.google.com', 52 'port': None, 53 'hostport': 'www.google.com', 54 'path': '/index.html', 55 'userinfo': '', 56 'username': '', 57 'password': '', 58 'params': {}, 59 'query': {}, 60 'relative': False 61 }, 62 "http://www.google.com/index.html;param1=value1" : { 63 'uri': 'http://www.google.com/index.html;param1=value1', 64 'scheme': 'http', 65 'hostname': 'www.google.com', 66 'port': None, 67 'hostport': 'www.google.com', 68 'path': '/index.html', 69 'userinfo': '', 70 'username': '', 71 'password': '', 72 'params': { 73 'param1': 'value1' 74 }, 75 'query': {}, 76 'relative': False 77 }, 78 "http://www.example.org/index.html?param1=value1" : { 79 'uri': 'http://www.example.org/index.html?param1=value1', 80 'scheme': 'http', 81 'hostname': 'www.example.org', 82 'port': None, 83 'hostport': 'www.example.org', 84 'path': '/index.html', 85 'userinfo': '', 86 'username': '', 87 'password': '', 88 'params': {}, 89 'query': { 90 'param1': 'value1' 91 }, 92 'relative': False 93 }, 94 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2" : { 95 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2', 96 'scheme': 'http', 97 'hostname': 'www.example.org', 98 'port': None, 99 'hostport': 'www.example.org', 100 'path': '/index.html', 101 'userinfo': '', 102 'username': '', 103 'password': '', 104 'params': { 105 'param2': 'value2' 106 }, 107 'query': { 108 'qparam1': 'qvalue1' 109 }, 110 'relative': False 111 }, 112 # Check that trailing semicolons are handled correctly 113 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2;" : { 114 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2', 115 'scheme': 'http', 116 'hostname': 'www.example.org', 117 'port': None, 118 'hostport': 'www.example.org', 119 'path': '/index.html', 120 'userinfo': '', 121 'username': '', 122 'password': '', 123 'params': { 124 'param2': 'value2' 125 }, 126 'query': { 127 'qparam1': 'qvalue1' 128 }, 129 'relative': False 130 }, 131 "http://www.example.com:8080/index.html" : { 132 'uri': 'http://www.example.com:8080/index.html', 133 'scheme': 'http', 134 'hostname': 'www.example.com', 135 'port': 8080, 136 'hostport': 'www.example.com:8080', 137 'path': '/index.html', 138 'userinfo': '', 139 'username': '', 140 'password': '', 141 'params': {}, 142 'query': {}, 143 'relative': False 144 }, 145 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : { 146 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg', 147 'scheme': 'cvs', 148 'hostname': 'cvs.handhelds.org', 149 'port': None, 150 'hostport': 'cvs.handhelds.org', 151 'path': '/cvs', 152 'userinfo': 'anoncvs', 153 'username': 'anoncvs', 154 'password': '', 155 'params': { 156 'module': 'familiar/dist/ipkg' 157 }, 158 'query': {}, 159 'relative': False 160 }, 161 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": { 162 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg', 163 'scheme': 'cvs', 164 'hostname': 'cvs.handhelds.org', 165 'port': None, 166 'hostport': 'cvs.handhelds.org', 167 'path': '/cvs', 168 'userinfo': 'anoncvs:anonymous', 169 'username': 'anoncvs', 170 'password': 'anonymous', 171 'params': collections.OrderedDict([ 172 ('tag', 'V0-99-81'), 173 ('module', 'familiar/dist/ipkg') 174 ]), 175 'query': {}, 176 'relative': False 177 }, 178 "file://example.diff": { # NOTE: Not RFC compliant! 179 'uri': 'file:example.diff', 180 'scheme': 'file', 181 'hostname': '', 182 'port': None, 183 'hostport': '', 184 'path': 'example.diff', 185 'userinfo': '', 186 'username': '', 187 'password': '', 188 'params': {}, 189 'query': {}, 190 'relative': True 191 }, 192 "file:example.diff": { # NOTE: RFC compliant version of the former 193 'uri': 'file:example.diff', 194 'scheme': 'file', 195 'hostname': '', 196 'port': None, 197 'hostport': '', 198 'path': 'example.diff', 199 'userinfo': '', 200 'userinfo': '', 201 'username': '', 202 'password': '', 203 'params': {}, 204 'query': {}, 205 'relative': True 206 }, 207 "file:///tmp/example.diff": { 208 'uri': 'file:///tmp/example.diff', 209 'scheme': 'file', 210 'hostname': '', 211 'port': None, 212 'hostport': '', 213 'path': '/tmp/example.diff', 214 'userinfo': '', 215 'userinfo': '', 216 'username': '', 217 'password': '', 218 'params': {}, 219 'query': {}, 220 'relative': False 221 }, 222 "git:///path/example.git": { 223 'uri': 'git:///path/example.git', 224 'scheme': 'git', 225 'hostname': '', 226 'port': None, 227 'hostport': '', 228 'path': '/path/example.git', 229 'userinfo': '', 230 'userinfo': '', 231 'username': '', 232 'password': '', 233 'params': {}, 234 'query': {}, 235 'relative': False 236 }, 237 "git:path/example.git": { 238 'uri': 'git:path/example.git', 239 'scheme': 'git', 240 'hostname': '', 241 'port': None, 242 'hostport': '', 243 'path': 'path/example.git', 244 'userinfo': '', 245 'userinfo': '', 246 'username': '', 247 'password': '', 248 'params': {}, 249 'query': {}, 250 'relative': True 251 }, 252 "git://example.net/path/example.git": { 253 'uri': 'git://example.net/path/example.git', 254 'scheme': 'git', 255 'hostname': 'example.net', 256 'port': None, 257 'hostport': 'example.net', 258 'path': '/path/example.git', 259 'userinfo': '', 260 'userinfo': '', 261 'username': '', 262 'password': '', 263 'params': {}, 264 'query': {}, 265 'relative': False 266 }, 267 "git://tfs-example.org:22/tfs/example%20path/example.git": { 268 'uri': 'git://tfs-example.org:22/tfs/example%20path/example.git', 269 'scheme': 'git', 270 'hostname': 'tfs-example.org', 271 'port': 22, 272 'hostport': 'tfs-example.org:22', 273 'path': '/tfs/example path/example.git', 274 'userinfo': '', 275 'userinfo': '', 276 'username': '', 277 'password': '', 278 'params': {}, 279 'query': {}, 280 'relative': False 281 }, 282 "http://somesite.net;someparam=1": { 283 'uri': 'http://somesite.net;someparam=1', 284 'scheme': 'http', 285 'hostname': 'somesite.net', 286 'port': None, 287 'hostport': 'somesite.net', 288 'path': '', 289 'userinfo': '', 290 'userinfo': '', 291 'username': '', 292 'password': '', 293 'params': {"someparam" : "1"}, 294 'query': {}, 295 'relative': False 296 }, 297 "file://somelocation;someparam=1": { 298 'uri': 'file:somelocation;someparam=1', 299 'scheme': 'file', 300 'hostname': '', 301 'port': None, 302 'hostport': '', 303 'path': 'somelocation', 304 'userinfo': '', 305 'userinfo': '', 306 'username': '', 307 'password': '', 308 'params': {"someparam" : "1"}, 309 'query': {}, 310 'relative': True 311 }, 312 "https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip": { 313 'uri': 'https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip', 314 'scheme': 'https', 315 'hostname': 'www.innodisk.com', 316 'port': None, 317 'hostport': 'www.innodisk.com', 318 'path': '/Download_file', 319 'userinfo': '', 320 'userinfo': '', 321 'username': '', 322 'password': '', 323 'params': {"downloadfilename" : "EGPL-T101.zip"}, 324 'query': {"9BE0BF6657": None}, 325 'relative': False 326 } 327 328 } 329 330 def test_uri(self): 331 for test_uri, ref in self.test_uris.items(): 332 uri = URI(test_uri) 333 334 self.assertEqual(str(uri), ref['uri']) 335 336 # expected attributes 337 self.assertEqual(uri.scheme, ref['scheme']) 338 339 self.assertEqual(uri.userinfo, ref['userinfo']) 340 self.assertEqual(uri.username, ref['username']) 341 self.assertEqual(uri.password, ref['password']) 342 343 self.assertEqual(uri.hostname, ref['hostname']) 344 self.assertEqual(uri.port, ref['port']) 345 self.assertEqual(uri.hostport, ref['hostport']) 346 347 self.assertEqual(uri.path, ref['path']) 348 self.assertEqual(uri.params, ref['params']) 349 350 self.assertEqual(uri.relative, ref['relative']) 351 352 def test_dict(self): 353 for test in self.test_uris.values(): 354 uri = URI() 355 356 self.assertEqual(uri.scheme, '') 357 self.assertEqual(uri.userinfo, '') 358 self.assertEqual(uri.username, '') 359 self.assertEqual(uri.password, '') 360 self.assertEqual(uri.hostname, '') 361 self.assertEqual(uri.port, None) 362 self.assertEqual(uri.path, '') 363 self.assertEqual(uri.params, {}) 364 365 366 uri.scheme = test['scheme'] 367 self.assertEqual(uri.scheme, test['scheme']) 368 369 uri.userinfo = test['userinfo'] 370 self.assertEqual(uri.userinfo, test['userinfo']) 371 self.assertEqual(uri.username, test['username']) 372 self.assertEqual(uri.password, test['password']) 373 374 # make sure changing the values doesn't do anything unexpected 375 uri.username = 'changeme' 376 self.assertEqual(uri.username, 'changeme') 377 self.assertEqual(uri.password, test['password']) 378 uri.password = 'insecure' 379 self.assertEqual(uri.username, 'changeme') 380 self.assertEqual(uri.password, 'insecure') 381 382 # reset back after our trickery 383 uri.userinfo = test['userinfo'] 384 self.assertEqual(uri.userinfo, test['userinfo']) 385 self.assertEqual(uri.username, test['username']) 386 self.assertEqual(uri.password, test['password']) 387 388 uri.hostname = test['hostname'] 389 self.assertEqual(uri.hostname, test['hostname']) 390 self.assertEqual(uri.hostport, test['hostname']) 391 392 uri.port = test['port'] 393 self.assertEqual(uri.port, test['port']) 394 self.assertEqual(uri.hostport, test['hostport']) 395 396 uri.path = test['path'] 397 self.assertEqual(uri.path, test['path']) 398 399 uri.params = test['params'] 400 self.assertEqual(uri.params, test['params']) 401 402 uri.query = test['query'] 403 self.assertEqual(uri.query, test['query']) 404 405 self.assertEqual(str(uri), test['uri']) 406 407 uri.params = {} 408 self.assertEqual(uri.params, {}) 409 self.assertEqual(str(uri), (str(uri).split(";"))[0]) 410 411class FetcherTest(unittest.TestCase): 412 413 def setUp(self): 414 self.origdir = os.getcwd() 415 self.d = bb.data.init() 416 self.tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-") 417 self.dldir = os.path.join(self.tempdir, "download") 418 os.mkdir(self.dldir) 419 self.d.setVar("DL_DIR", self.dldir) 420 self.unpackdir = os.path.join(self.tempdir, "unpacked") 421 os.mkdir(self.unpackdir) 422 persistdir = os.path.join(self.tempdir, "persistdata") 423 self.d.setVar("PERSISTENT_DIR", persistdir) 424 425 def tearDown(self): 426 os.chdir(self.origdir) 427 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes": 428 print("Not cleaning up %s. Please remove manually." % self.tempdir) 429 else: 430 bb.process.run('chmod u+rw -R %s' % self.tempdir) 431 bb.utils.prunedir(self.tempdir) 432 433 def git(self, cmd, cwd=None): 434 if isinstance(cmd, str): 435 cmd = 'git -c safe.bareRepository=all ' + cmd 436 else: 437 cmd = ['git', '-c', 'safe.bareRepository=all'] + cmd 438 if cwd is None: 439 cwd = self.gitdir 440 return bb.process.run(cmd, cwd=cwd)[0] 441 442 def git_init(self, cwd=None): 443 self.git('init', cwd=cwd) 444 # Explicitly set initial branch to master as 445 # a common setup is to use other default 446 # branch than master. 447 self.git(['checkout', '-b', 'master'], cwd=cwd) 448 449 try: 450 self.git(['config', 'user.email'], cwd=cwd) 451 except bb.process.ExecutionError: 452 self.git(['config', 'user.email', 'you@example.com'], cwd=cwd) 453 454 try: 455 self.git(['config', 'user.name'], cwd=cwd) 456 except bb.process.ExecutionError: 457 self.git(['config', 'user.name', 'Your Name'], cwd=cwd) 458 459class MirrorUriTest(FetcherTest): 460 461 replaceuris = { 462 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/") 463 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz", 464 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 465 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 466 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 467 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 468 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http") 469 : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 470 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake") 471 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890", 472 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache") 473 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 474 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/") 475 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 476 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3") 477 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", 478 ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz") 479 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", 480 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist") 481 : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", 482 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/") 483 : "file:///somepath/downloads/subversion-1.7.1.tar.bz2", 484 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 485 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 486 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 487 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 488 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http") 489 : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 490 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org") 491 : "http://somewhere2.org/somefile_1.2.3.tar.gz", 492 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/") 493 : "http://somewhere2.org/somefile_1.2.3.tar.gz", 494 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http") 495 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 496 ("git://user1@someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://user2@git.openembedded.org/bitbake;protocol=http") 497 : "git://user2@git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 498 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=git;branch=master", "git://someserver.org/bitbake", "git://someotherserver.org/bitbake;protocol=https") 499 : "git://someotherserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=https;branch=master", 500 ("gitsm://git.qemu.org/git/seabios.git/;protocol=https;name=roms/seabios;subpath=roms/seabios;bareclone=1;nobranch=1;rev=1234567890123456789012345678901234567890", "gitsm://.*/.*", "http://petalinux.xilinx.com/sswreleases/rel-v${XILINX_VER_MAIN}/downloads") : "http://petalinux.xilinx.com/sswreleases/rel-v%24%7BXILINX_VER_MAIN%7D/downloads/git2_git.qemu.org.git.seabios.git..tar.gz", 501 ("https://somewhere.org/example/1.0.0/example;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/PATH") 502 : "file:///mirror/example/1.0.0/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", 503 ("https://somewhere.org/example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/some-example-1.0.0.tgz") 504 : "file:///mirror/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", 505 506 #Renaming files doesn't work 507 #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz" 508 #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 509 } 510 511 mirrorvar = "http://.*/.* file:///somepath/downloads/ " \ 512 "git://someserver.org/bitbake git://git.openembedded.org/bitbake " \ 513 "https://.*/.* file:///someotherpath/downloads/ " \ 514 "http://.*/.* file:///someotherpath/downloads/" 515 516 def test_urireplace(self): 517 self.d.setVar("FILESPATH", ".") 518 for k, v in self.replaceuris.items(): 519 ud = bb.fetch.FetchData(k[0], self.d) 520 ud.setup_localpath(self.d) 521 mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2])) 522 newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d) 523 self.assertEqual([v], newuris) 524 525 def test_urilist1(self): 526 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 527 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) 528 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 529 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz']) 530 531 def test_urilist2(self): 532 # Catch https:// -> files:// bug 533 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 534 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) 535 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 536 self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz']) 537 538 def test_mirror_of_mirror(self): 539 # Test if mirror of a mirror works 540 mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/" 541 mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/" 542 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 543 mirrors = bb.fetch2.mirror_from_string(mirrorvar) 544 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 545 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 546 'file:///someotherpath/downloads/bitbake-1.0.tar.gz', 547 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz', 548 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz']) 549 550 recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ " \ 551 "https://.*/[^/]* https://BBBB/B/B/B/" 552 553 def test_recursive(self): 554 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 555 mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar) 556 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 557 self.assertEqual(uris, ['http://AAAA/A/A/A/bitbake/bitbake-1.0.tar.gz', 558 'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz', 559 'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz']) 560 561 562class GitDownloadDirectoryNamingTest(FetcherTest): 563 def setUp(self): 564 super(GitDownloadDirectoryNamingTest, self).setUp() 565 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 566 self.recipe_dir = "git.openembedded.org.bitbake" 567 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 568 self.mirror_dir = "github.com.openembedded.bitbake.git" 569 570 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 571 572 def setup_mirror_rewrite(self): 573 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 574 575 @skipIfNoNetwork() 576 def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self): 577 self.setup_mirror_rewrite() 578 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 579 580 fetcher.download() 581 582 dir = os.listdir(self.dldir + "/git2") 583 self.assertIn(self.recipe_dir, dir) 584 585 @skipIfNoNetwork() 586 def test_that_directory_exists_for_mirrored_url_and_recipe_url_when_mirroring_is_used(self): 587 self.setup_mirror_rewrite() 588 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 589 590 fetcher.download() 591 592 dir = os.listdir(self.dldir + "/git2") 593 self.assertIn(self.mirror_dir, dir) 594 self.assertIn(self.recipe_dir, dir) 595 596 @skipIfNoNetwork() 597 def test_that_recipe_directory_and_mirrored_directory_exists_when_mirroring_is_used_and_the_mirrored_directory_already_exists(self): 598 self.setup_mirror_rewrite() 599 fetcher = bb.fetch.Fetch([self.mirror_url], self.d) 600 fetcher.download() 601 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 602 603 fetcher.download() 604 605 dir = os.listdir(self.dldir + "/git2") 606 self.assertIn(self.mirror_dir, dir) 607 self.assertIn(self.recipe_dir, dir) 608 609 610class TarballNamingTest(FetcherTest): 611 def setUp(self): 612 super(TarballNamingTest, self).setUp() 613 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 614 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz" 615 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 616 self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz" 617 618 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1') 619 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 620 621 def setup_mirror_rewrite(self): 622 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 623 624 @skipIfNoNetwork() 625 def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self): 626 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 627 628 fetcher.download() 629 630 dir = os.listdir(self.dldir) 631 self.assertIn(self.recipe_tarball, dir) 632 633 @skipIfNoNetwork() 634 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self): 635 self.setup_mirror_rewrite() 636 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 637 638 fetcher.download() 639 640 dir = os.listdir(self.dldir) 641 self.assertIn(self.mirror_tarball, dir) 642 643 644class GitShallowTarballNamingTest(FetcherTest): 645 def setUp(self): 646 super(GitShallowTarballNamingTest, self).setUp() 647 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 648 self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz" 649 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 650 self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz" 651 652 self.d.setVar('BB_GIT_SHALLOW', '1') 653 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 654 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 655 656 def setup_mirror_rewrite(self): 657 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 658 659 @skipIfNoNetwork() 660 def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self): 661 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 662 663 fetcher.download() 664 665 dir = os.listdir(self.dldir) 666 self.assertIn(self.recipe_tarball, dir) 667 668 @skipIfNoNetwork() 669 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self): 670 self.setup_mirror_rewrite() 671 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 672 673 fetcher.download() 674 675 dir = os.listdir(self.dldir) 676 self.assertIn(self.mirror_tarball, dir) 677 678 679class CleanTarballTest(FetcherTest): 680 def setUp(self): 681 super(CleanTarballTest, self).setUp() 682 self.recipe_url = "git://git.openembedded.org/bitbake;protocol=https" 683 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz" 684 685 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1') 686 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 687 688 @skipIfNoNetwork() 689 def test_that_the_tarball_contents_does_not_leak_info(self): 690 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 691 692 fetcher.download() 693 694 fetcher.unpack(self.unpackdir) 695 mtime = bb.process.run('git log --all -1 --format=%ct', 696 cwd=os.path.join(self.unpackdir, 'git')) 697 self.assertEqual(len(mtime), 2) 698 mtime = int(mtime[0]) 699 700 archive = tarfile.open(os.path.join(self.dldir, self.recipe_tarball)) 701 self.assertNotEqual(len(archive.members), 0) 702 for member in archive.members: 703 if member.name == ".": 704 continue 705 self.assertEqual(member.uname, 'oe', "user name for %s differs" % member.name) 706 self.assertEqual(member.uid, 0, "uid for %s differs" % member.name) 707 self.assertEqual(member.gname, 'oe', "group name for %s differs" % member.name) 708 self.assertEqual(member.gid, 0, "gid for %s differs" % member.name) 709 self.assertEqual(member.mtime, mtime, "mtime for %s differs" % member.name) 710 711 712class FetcherLocalTest(FetcherTest): 713 def setUp(self): 714 def touch(fn): 715 with open(fn, 'a'): 716 os.utime(fn, None) 717 718 super(FetcherLocalTest, self).setUp() 719 self.localsrcdir = os.path.join(self.tempdir, 'localsrc') 720 os.makedirs(self.localsrcdir) 721 touch(os.path.join(self.localsrcdir, 'a')) 722 touch(os.path.join(self.localsrcdir, 'b')) 723 os.makedirs(os.path.join(self.localsrcdir, 'dir')) 724 touch(os.path.join(self.localsrcdir, 'dir', 'c')) 725 touch(os.path.join(self.localsrcdir, 'dir', 'd')) 726 os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir')) 727 touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e')) 728 touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device')) 729 bb.process.run('tar cf archive.tar -C dir .', cwd=self.localsrcdir) 730 bb.process.run('tar czf archive.tar.gz -C dir .', cwd=self.localsrcdir) 731 bb.process.run('tar cjf archive.tar.bz2 -C dir .', cwd=self.localsrcdir) 732 self.d.setVar("FILESPATH", self.localsrcdir) 733 734 def fetchUnpack(self, uris): 735 fetcher = bb.fetch.Fetch(uris, self.d) 736 fetcher.download() 737 fetcher.unpack(self.unpackdir) 738 flst = [] 739 for root, dirs, files in os.walk(self.unpackdir): 740 for f in files: 741 flst.append(os.path.relpath(os.path.join(root, f), self.unpackdir)) 742 flst.sort() 743 return flst 744 745 def test_local_checksum_fails_no_file(self): 746 self.d.setVar("SRC_URI", "file://404") 747 with self.assertRaises(bb.BBHandledException): 748 bb.fetch.get_checksum_file_list(self.d) 749 750 def test_local(self): 751 tree = self.fetchUnpack(['file://a', 'file://dir/c']) 752 self.assertEqual(tree, ['a', 'dir/c']) 753 754 def test_local_backslash(self): 755 tree = self.fetchUnpack([r'file://backslash\x2dsystemd-unit.device']) 756 self.assertEqual(tree, [r'backslash\x2dsystemd-unit.device']) 757 758 def test_local_wildcard(self): 759 with self.assertRaises(bb.fetch2.ParameterError): 760 tree = self.fetchUnpack(['file://a', 'file://dir/*']) 761 762 def test_local_dir(self): 763 tree = self.fetchUnpack(['file://a', 'file://dir']) 764 self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e']) 765 766 def test_local_subdir(self): 767 tree = self.fetchUnpack(['file://dir/subdir']) 768 self.assertEqual(tree, ['dir/subdir/e']) 769 770 def test_local_subdir_file(self): 771 tree = self.fetchUnpack(['file://dir/subdir/e']) 772 self.assertEqual(tree, ['dir/subdir/e']) 773 774 def test_local_subdirparam(self): 775 tree = self.fetchUnpack(['file://a;subdir=bar', 'file://dir;subdir=foo/moo']) 776 self.assertEqual(tree, ['bar/a', 'foo/moo/dir/c', 'foo/moo/dir/d', 'foo/moo/dir/subdir/e']) 777 778 def test_local_deepsubdirparam(self): 779 tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar']) 780 self.assertEqual(tree, ['bar/dir/subdir/e']) 781 782 def test_local_absolutedir(self): 783 # Unpacking to an absolute path that is a subdirectory of the root 784 # should work 785 tree = self.fetchUnpack(['file://a;subdir=%s' % os.path.join(self.unpackdir, 'bar')]) 786 787 # Unpacking to an absolute path outside of the root should fail 788 with self.assertRaises(bb.fetch2.UnpackError): 789 self.fetchUnpack(['file://a;subdir=/bin/sh']) 790 791 def test_local_striplevel(self): 792 tree = self.fetchUnpack(['file://archive.tar;subdir=bar;striplevel=1']) 793 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 794 795 def test_local_striplevel_gzip(self): 796 tree = self.fetchUnpack(['file://archive.tar.gz;subdir=bar;striplevel=1']) 797 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 798 799 def test_local_striplevel_bzip2(self): 800 tree = self.fetchUnpack(['file://archive.tar.bz2;subdir=bar;striplevel=1']) 801 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 802 803 def dummyGitTest(self, suffix): 804 # Create dummy local Git repo 805 src_dir = tempfile.mkdtemp(dir=self.tempdir, 806 prefix='gitfetch_localusehead_') 807 self.gitdir = os.path.abspath(src_dir) 808 self.git_init() 809 self.git(['commit', '--allow-empty', '-m', 'Dummy commit']) 810 # Use other branch than master 811 self.git(['checkout', '-b', 'my-devel']) 812 self.git(['commit', '--allow-empty', '-m', 'Dummy commit 2']) 813 orig_rev = self.git(['rev-parse', 'HEAD']).strip() 814 815 # Fetch and check revision 816 self.d.setVar("SRCREV", "AUTOINC") 817 self.d.setVar("__BBSRCREV_SEEN", "1") 818 url = "git://" + self.gitdir + ";branch=master;protocol=file;" + suffix 819 fetcher = bb.fetch.Fetch([url], self.d) 820 fetcher.download() 821 fetcher.unpack(self.unpackdir) 822 unpack_rev = self.git(['rev-parse', 'HEAD'], 823 cwd=os.path.join(self.unpackdir, 'git')).strip() 824 self.assertEqual(orig_rev, unpack_rev) 825 826 def test_local_gitfetch_usehead(self): 827 self.dummyGitTest("usehead=1") 828 829 def test_local_gitfetch_usehead_withname(self): 830 self.dummyGitTest("usehead=1;name=newName") 831 832 def test_local_gitfetch_shared(self): 833 self.dummyGitTest("usehead=1;name=sharedName") 834 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 835 self.assertTrue(os.path.exists(alt)) 836 837 def test_local_gitfetch_noshared(self): 838 self.d.setVar('BB_GIT_NOSHARED', '1') 839 self.unpackdir += '_noshared' 840 self.dummyGitTest("usehead=1;name=noSharedName") 841 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 842 self.assertFalse(os.path.exists(alt)) 843 844class FetcherNoNetworkTest(FetcherTest): 845 def setUp(self): 846 super().setUp() 847 # all test cases are based on not having network 848 self.d.setVar("BB_NO_NETWORK", "1") 849 850 def test_missing(self): 851 string = "this is a test file\n".encode("utf-8") 852 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 853 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 854 855 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 856 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 857 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 858 with self.assertRaises(bb.fetch2.NetworkAccess): 859 fetcher.download() 860 861 def test_valid_missing_donestamp(self): 862 # create the file in the download directory with correct hash 863 string = "this is a test file\n".encode("utf-8") 864 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb") as f: 865 f.write(string) 866 867 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 868 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 869 870 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 871 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 872 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 873 fetcher.download() 874 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 875 876 def test_invalid_missing_donestamp(self): 877 # create an invalid file in the download directory with incorrect hash 878 string = "this is a test file\n".encode("utf-8") 879 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 880 pass 881 882 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 883 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 884 885 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 886 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 887 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 888 with self.assertRaises(bb.fetch2.NetworkAccess): 889 fetcher.download() 890 # the existing file should not exist or should have be moved to "bad-checksum" 891 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 892 893 def test_nochecksums_missing(self): 894 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 895 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 896 # ssh fetch does not support checksums 897 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 898 # attempts to download with missing donestamp 899 with self.assertRaises(bb.fetch2.NetworkAccess): 900 fetcher.download() 901 902 def test_nochecksums_missing_donestamp(self): 903 # create a file in the download directory 904 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 905 pass 906 907 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 908 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 909 # ssh fetch does not support checksums 910 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 911 # attempts to download with missing donestamp 912 with self.assertRaises(bb.fetch2.NetworkAccess): 913 fetcher.download() 914 915 def test_nochecksums_has_donestamp(self): 916 # create a file in the download directory with the donestamp 917 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 918 pass 919 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"): 920 pass 921 922 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 923 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 924 # ssh fetch does not support checksums 925 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 926 # should not fetch 927 fetcher.download() 928 # both files should still exist 929 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 930 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 931 932 def test_nochecksums_missing_has_donestamp(self): 933 # create a file in the download directory with the donestamp 934 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"): 935 pass 936 937 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 938 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 939 # ssh fetch does not support checksums 940 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 941 with self.assertRaises(bb.fetch2.NetworkAccess): 942 fetcher.download() 943 # both files should still exist 944 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 945 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 946 947class FetcherNetworkTest(FetcherTest): 948 @skipIfNoNetwork() 949 def test_fetch(self): 950 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) 951 fetcher.download() 952 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 953 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892) 954 self.d.setVar("BB_NO_NETWORK", "1") 955 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) 956 fetcher.download() 957 fetcher.unpack(self.unpackdir) 958 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9) 959 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9) 960 961 @skipIfNoNetwork() 962 def test_fetch_mirror(self): 963 self.d.setVar("MIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 964 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 965 fetcher.download() 966 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 967 968 @skipIfNoNetwork() 969 def test_fetch_mirror_of_mirror(self): 970 self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ http://invalid2.yoctoproject.org/.* https://downloads.yoctoproject.org/releases/bitbake") 971 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 972 fetcher.download() 973 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 974 975 @skipIfNoNetwork() 976 def test_fetch_file_mirror_of_mirror(self): 977 self.d.setVar("FILESPATH", ".") 978 self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ file:///some1where/.* file://some2where/ file://some2where/.* https://downloads.yoctoproject.org/releases/bitbake") 979 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 980 os.mkdir(self.dldir + "/some2where") 981 fetcher.download() 982 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 983 984 @skipIfNoNetwork() 985 def test_fetch_premirror(self): 986 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 987 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 988 fetcher.download() 989 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 990 991 @skipIfNoNetwork() 992 def test_fetch_specify_downloadfilename(self): 993 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d) 994 fetcher.download() 995 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749) 996 997 @skipIfNoNetwork() 998 def test_fetch_premirror_specify_downloadfilename_regex_uri(self): 999 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake/") 1000 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 1001 fetcher.download() 1002 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1003 1004 @skipIfNoNetwork() 1005 # BZ13039 1006 def test_fetch_premirror_specify_downloadfilename_specific_uri(self): 1007 self.d.setVar("PREMIRRORS", "http://invalid.yoctoproject.org/releases/bitbake https://downloads.yoctoproject.org/releases/bitbake") 1008 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 1009 fetcher.download() 1010 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1011 1012 @skipIfNoNetwork() 1013 def test_fetch_premirror_use_downloadfilename_to_fetch(self): 1014 # Ensure downloadfilename is used when fetching from premirror. 1015 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 1016 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 1017 fetcher.download() 1018 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1019 1020 @skipIfNoNetwork() 1021 def gitfetcher(self, url1, url2): 1022 def checkrevision(self, fetcher): 1023 fetcher.unpack(self.unpackdir) 1024 revision = self.git(['rev-parse', 'HEAD'], 1025 cwd=os.path.join(self.unpackdir, 'git')).strip() 1026 self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5") 1027 1028 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") 1029 self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5") 1030 fetcher = bb.fetch.Fetch([url1], self.d) 1031 fetcher.download() 1032 checkrevision(self, fetcher) 1033 # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works 1034 bb.utils.prunedir(self.dldir + "/git2/") 1035 bb.utils.prunedir(self.unpackdir) 1036 self.d.setVar("BB_NO_NETWORK", "1") 1037 fetcher = bb.fetch.Fetch([url2], self.d) 1038 fetcher.download() 1039 checkrevision(self, fetcher) 1040 1041 @skipIfNoNetwork() 1042 def test_gitfetch(self): 1043 url1 = url2 = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 1044 self.gitfetcher(url1, url2) 1045 1046 @skipIfNoNetwork() 1047 def test_gitfetch_goodsrcrev(self): 1048 # SRCREV is set but matches rev= parameter 1049 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https" 1050 self.gitfetcher(url1, url2) 1051 1052 @skipIfNoNetwork() 1053 def test_gitfetch_badsrcrev(self): 1054 # SRCREV is set but does not match rev= parameter 1055 url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https" 1056 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) 1057 1058 @skipIfNoNetwork() 1059 def test_gitfetch_tagandrev(self): 1060 # SRCREV is set but does not match rev= parameter 1061 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5;protocol=https" 1062 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) 1063 1064 @skipIfNoNetwork() 1065 def test_gitfetch_usehead(self): 1066 # Since self.gitfetcher() sets SRCREV we expect this to override 1067 # `usehead=1' and instead fetch the specified SRCREV. See 1068 # test_local_gitfetch_usehead() for a positive use of the usehead 1069 # feature. 1070 url = "git://git.openembedded.org/bitbake;usehead=1;branch=master;protocol=https" 1071 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url) 1072 1073 @skipIfNoNetwork() 1074 def test_gitfetch_usehead_withname(self): 1075 # Since self.gitfetcher() sets SRCREV we expect this to override 1076 # `usehead=1' and instead fetch the specified SRCREV. See 1077 # test_local_gitfetch_usehead() for a positive use of the usehead 1078 # feature. 1079 url = "git://git.openembedded.org/bitbake;usehead=1;name=newName;branch=master;protocol=https" 1080 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url) 1081 1082 @skipIfNoNetwork() 1083 def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self): 1084 recipeurl = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 1085 mirrorurl = "git://someserver.org/bitbake;branch=master;protocol=https" 1086 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake") 1087 self.gitfetcher(recipeurl, mirrorurl) 1088 1089 @skipIfNoNetwork() 1090 def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self): 1091 recipeurl = "git://someserver.org/bitbake;branch=master;protocol=https" 1092 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake") 1093 self.gitfetcher(recipeurl, recipeurl) 1094 1095 @skipIfNoNetwork() 1096 def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self): 1097 realurl = "https://git.openembedded.org/bitbake" 1098 recipeurl = "git://someserver.org/bitbake;protocol=https" 1099 self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git") 1100 os.chdir(self.tempdir) 1101 self.git(['clone', realurl, self.sourcedir], cwd=self.tempdir) 1102 self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file" % (recipeurl, self.sourcedir)) 1103 self.gitfetcher(recipeurl, recipeurl) 1104 1105 @skipIfNoNetwork() 1106 def test_git_submodule(self): 1107 # URL with ssh submodules 1108 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7;branch=master;protocol=https" 1109 # Original URL (comment this if you have ssh access to git.yoctoproject.org) 1110 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee;branch=master;protocol=https" 1111 fetcher = bb.fetch.Fetch([url], self.d) 1112 fetcher.download() 1113 # Previous cwd has been deleted 1114 os.chdir(os.path.dirname(self.unpackdir)) 1115 fetcher.unpack(self.unpackdir) 1116 1117 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1118 self.assertTrue(os.path.exists(repo_path), msg='Unpacked repository missing') 1119 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake')), msg='bitbake submodule missing') 1120 self.assertFalse(os.path.exists(os.path.join(repo_path, 'na')), msg='uninitialized submodule present') 1121 1122 # Only when we're running the extended test with a submodule's submodule, can we check this. 1123 if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')): 1124 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing') 1125 1126 @skipIfNoNetwork() 1127 def test_git_submodule_restricted_network_premirrors(self): 1128 # this test is to ensure that premirrors will be tried in restricted network 1129 # that is, BB_ALLOWED_NETWORKS does not contain the domain the url uses 1130 url = "gitsm://github.com/grpc/grpc.git;protocol=https;name=grpc;branch=v1.60.x;rev=0ef13a7555dbaadd4633399242524129eef5e231" 1131 # create a download directory to be used as premirror later 1132 tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-") 1133 dl_premirror = os.path.join(tempdir, "download-premirror") 1134 os.mkdir(dl_premirror) 1135 self.d.setVar("DL_DIR", dl_premirror) 1136 fetcher = bb.fetch.Fetch([url], self.d) 1137 fetcher.download() 1138 # now use the premirror in restricted network 1139 self.d.setVar("DL_DIR", self.dldir) 1140 self.d.setVar("PREMIRRORS", "gitsm://.*/.* gitsm://%s/git2/MIRRORNAME;protocol=file" % dl_premirror) 1141 self.d.setVar("BB_ALLOWED_NETWORKS", "*.some.domain") 1142 fetcher = bb.fetch.Fetch([url], self.d) 1143 fetcher.download() 1144 1145 @skipIfNoNetwork() 1146 def test_git_submodule_dbus_broker(self): 1147 # The following external repositories have show failures in fetch and unpack operations 1148 # We want to avoid regressions! 1149 url = "gitsm://github.com/bus1/dbus-broker;protocol=https;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main" 1150 fetcher = bb.fetch.Fetch([url], self.d) 1151 fetcher.download() 1152 # Previous cwd has been deleted 1153 os.chdir(os.path.dirname(self.unpackdir)) 1154 fetcher.unpack(self.unpackdir) 1155 1156 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1157 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-dvar/config')), msg='Missing submodule config "subprojects/c-dvar"') 1158 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-list/config')), msg='Missing submodule config "subprojects/c-list"') 1159 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-rbtree/config')), msg='Missing submodule config "subprojects/c-rbtree"') 1160 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"') 1161 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"') 1162 1163 @skipIfNoNetwork() 1164 def test_git_submodule_CLI11(self): 1165 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main" 1166 fetcher = bb.fetch.Fetch([url], self.d) 1167 fetcher.download() 1168 # Previous cwd has been deleted 1169 os.chdir(os.path.dirname(self.unpackdir)) 1170 fetcher.unpack(self.unpackdir) 1171 1172 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1173 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"') 1174 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') 1175 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') 1176 1177 @skipIfNoNetwork() 1178 def test_git_submodule_update_CLI11(self): 1179 """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """ 1180 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main" 1181 fetcher = bb.fetch.Fetch([url], self.d) 1182 fetcher.download() 1183 1184 # CLI11 that pulls in a newer nlohmann-json 1185 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main" 1186 fetcher = bb.fetch.Fetch([url], self.d) 1187 fetcher.download() 1188 # Previous cwd has been deleted 1189 os.chdir(os.path.dirname(self.unpackdir)) 1190 fetcher.unpack(self.unpackdir) 1191 1192 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1193 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"') 1194 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') 1195 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') 1196 1197 @skipIfNoNetwork() 1198 def test_git_submodule_aktualizr(self): 1199 url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=https;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44" 1200 fetcher = bb.fetch.Fetch([url], self.d) 1201 fetcher.download() 1202 # Previous cwd has been deleted 1203 os.chdir(os.path.dirname(self.unpackdir)) 1204 fetcher.unpack(self.unpackdir) 1205 1206 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1207 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule config "partial/extern/isotp-c/config"') 1208 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')), msg='Missing submodule config "partial/extern/isotp-c/modules/deps/bitfield-c/config"') 1209 self.assertTrue(os.path.exists(os.path.join(repo_path, 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of submodule isotp-c did not unpack properly") 1210 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule config "tests/tuf-test-vectors/config"') 1211 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"') 1212 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"') 1213 1214 @skipIfNoNetwork() 1215 def test_git_submodule_iotedge(self): 1216 """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """ 1217 1218 # This repository also has submodules where the module (name), path and url do not align 1219 url = "gitsm://github.com/azure/iotedge.git;protocol=https;rev=d76e0316c6f324345d77c48a83ce836d09392699;branch=main" 1220 fetcher = bb.fetch.Fetch([url], self.d) 1221 fetcher.download() 1222 # Previous cwd has been deleted 1223 os.chdir(os.path.dirname(self.unpackdir)) 1224 fetcher.unpack(self.unpackdir) 1225 1226 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1227 1228 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/README.md')), msg='Missing submodule checkout') 1229 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/ctest/README.md')), msg='Missing submodule checkout') 1230 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/testrunner/readme.md')), msg='Missing submodule checkout') 1231 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/readme.md')), msg='Missing submodule checkout') 1232 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') 1233 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') 1234 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/README.md')), msg='Missing submodule checkout') 1235 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/README.md')), msg='Missing submodule checkout') 1236 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/ctest/README.md')), msg='Missing submodule checkout') 1237 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/testrunner/readme.md')), msg='Missing submodule checkout') 1238 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/readme.md')), msg='Missing submodule checkout') 1239 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') 1240 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') 1241 1242 @skipIfNoNetwork() 1243 def test_git_submodule_reference_to_parent(self): 1244 self.recipe_url = "gitsm://github.com/gflags/gflags.git;protocol=https;branch=master" 1245 self.d.setVar("SRCREV", "14e1138441bbbb584160cb1c0a0426ec1bac35f1") 1246 with Timeout(60): 1247 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 1248 with self.assertRaises(bb.fetch2.FetchError): 1249 fetcher.download() 1250 1251class SVNTest(FetcherTest): 1252 def skipIfNoSvn(): 1253 import shutil 1254 if not shutil.which("svn"): 1255 return unittest.skip("svn not installed, tests being skipped") 1256 1257 if not shutil.which("svnadmin"): 1258 return unittest.skip("svnadmin not installed, tests being skipped") 1259 1260 return lambda f: f 1261 1262 @skipIfNoSvn() 1263 def setUp(self): 1264 """ Create a local repository """ 1265 1266 super(SVNTest, self).setUp() 1267 1268 # Create something we can fetch 1269 src_dir = tempfile.mkdtemp(dir=self.tempdir, 1270 prefix='svnfetch_srcdir_') 1271 src_dir = os.path.abspath(src_dir) 1272 bb.process.run("echo readme > README.md", cwd=src_dir) 1273 1274 # Store it in a local SVN repository 1275 repo_dir = tempfile.mkdtemp(dir=self.tempdir, 1276 prefix='svnfetch_localrepo_') 1277 repo_dir = os.path.abspath(repo_dir) 1278 bb.process.run("svnadmin create project", cwd=repo_dir) 1279 1280 self.repo_url = "file://%s/project" % repo_dir 1281 bb.process.run("svn import --non-interactive -m 'Initial import' %s %s/trunk" % (src_dir, self.repo_url), 1282 cwd=repo_dir) 1283 1284 bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir) 1285 # Github won't emulate SVN anymore (see https://github.blog/2023-01-20-sunsetting-subversion-support/) 1286 # Use still accessible svn repo (only trunk to avoid longer downloads) 1287 bb.process.run("svn propset svn:externals 'bitbake https://svn.apache.org/repos/asf/serf/trunk' .", 1288 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) 1289 bb.process.run("svn commit --non-interactive -m 'Add external'", 1290 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) 1291 1292 self.src_dir = src_dir 1293 self.repo_dir = repo_dir 1294 1295 @skipIfNoSvn() 1296 def tearDown(self): 1297 os.chdir(self.origdir) 1298 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes": 1299 print("Not cleaning up %s. Please remove manually." % self.tempdir) 1300 else: 1301 bb.utils.prunedir(self.tempdir) 1302 1303 @skipIfNoSvn() 1304 @skipIfNoNetwork() 1305 def test_noexternal_svn(self): 1306 # Always match the rev count from setUp (currently rev 2) 1307 url = "svn://%s;module=trunk;protocol=file;rev=2" % self.repo_url.replace('file://', '') 1308 fetcher = bb.fetch.Fetch([url], self.d) 1309 fetcher.download() 1310 os.chdir(os.path.dirname(self.unpackdir)) 1311 fetcher.unpack(self.unpackdir) 1312 1313 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") 1314 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") 1315 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should NOT exist") 1316 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should NOT exit") 1317 1318 @skipIfNoSvn() 1319 def test_external_svn(self): 1320 # Always match the rev count from setUp (currently rev 2) 1321 url = "svn://%s;module=trunk;protocol=file;externals=allowed;rev=2" % self.repo_url.replace('file://', '') 1322 fetcher = bb.fetch.Fetch([url], self.d) 1323 fetcher.download() 1324 os.chdir(os.path.dirname(self.unpackdir)) 1325 fetcher.unpack(self.unpackdir) 1326 1327 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") 1328 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") 1329 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should exist") 1330 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should exit") 1331 1332class TrustedNetworksTest(FetcherTest): 1333 def test_trusted_network(self): 1334 # Ensure trusted_network returns False when the host IS in the list. 1335 url = "git://Someserver.org/foo;rev=1;branch=master" 1336 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org") 1337 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1338 1339 def test_wild_trusted_network(self): 1340 # Ensure trusted_network returns true when the *.host IS in the list. 1341 url = "git://Someserver.org/foo;rev=1;branch=master" 1342 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1343 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1344 1345 def test_prefix_wild_trusted_network(self): 1346 # Ensure trusted_network returns true when the prefix matches *.host. 1347 url = "git://git.Someserver.org/foo;rev=1;branch=master" 1348 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1349 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1350 1351 def test_two_prefix_wild_trusted_network(self): 1352 # Ensure trusted_network returns true when the prefix matches *.host. 1353 url = "git://something.git.Someserver.org/foo;rev=1;branch=master" 1354 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1355 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1356 1357 def test_port_trusted_network(self): 1358 # Ensure trusted_network returns True, even if the url specifies a port. 1359 url = "git://someserver.org:8080/foo;rev=1;branch=master" 1360 self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org") 1361 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1362 1363 def test_untrusted_network(self): 1364 # Ensure trusted_network returns False when the host is NOT in the list. 1365 url = "git://someserver.org/foo;rev=1;branch=master" 1366 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org") 1367 self.assertFalse(bb.fetch.trusted_network(self.d, url)) 1368 1369 def test_wild_untrusted_network(self): 1370 # Ensure trusted_network returns False when the host is NOT in the list. 1371 url = "git://*.someserver.org/foo;rev=1;branch=master" 1372 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org") 1373 self.assertFalse(bb.fetch.trusted_network(self.d, url)) 1374 1375class URLHandle(unittest.TestCase): 1376 1377 datatable = { 1378 "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}), 1379 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}), 1380 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])), 1381 "git://git.openembedded.org/bitbake;branch=@foo;protocol=https" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo', 'protocol' : 'https'}), 1382 "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}), 1383 "https://somesite.com/somerepo.git;user=anyUser:idtoken=1234" : ('https', 'somesite.com', '/somerepo.git', '', '', {'user': 'anyUser:idtoken=1234'}), 1384 r'git://s.o-me_ONE:!#$%^&*()-_={}[]\|:?,.<>~`@git.openembedded.org/bitbake;branch=main;protocol=https': ('git', 'git.openembedded.org', '/bitbake', 's.o-me_ONE', r'!#$%^&*()-_={}[]\|:?,.<>~`', {'branch': 'main', 'protocol' : 'https'}), 1385 } 1386 # we require a pathname to encodeurl but users can still pass such urls to 1387 # decodeurl and we need to handle them 1388 decodedata = datatable.copy() 1389 decodedata.update({ 1390 "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}), 1391 "npmsw://some.registry.url;package=@pkg;version=latest": ('npmsw', 'some.registry.url', '/', '', '', {'package': '@pkg', 'version': 'latest'}), 1392 }) 1393 1394 def test_decodeurl(self): 1395 for k, v in self.decodedata.items(): 1396 result = bb.fetch.decodeurl(k) 1397 self.assertEqual(result, v) 1398 1399 def test_encodeurl(self): 1400 for k, v in self.datatable.items(): 1401 result = bb.fetch.encodeurl(v) 1402 self.assertEqual(result, k) 1403 1404class FetchLatestVersionTest(FetcherTest): 1405 1406 test_git_uris = { 1407 # version pattern "X.Y.Z" 1408 ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "", "") 1409 : "1.99.4", 1410 # version pattern "vX.Y" 1411 # mirror of git.infradead.org since network issues interfered with testing 1412 ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git;branch=master;protocol=https", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "", "") 1413 : "1.5.0", 1414 # version pattern "pkg_name-X.Y" 1415 # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing 1416 ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto;branch=master;protocol=https", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "", "") 1417 : "1.0", 1418 # version pattern "pkg_name-vX.Y.Z" 1419 ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git;branch=master;protocol=https", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "", "") 1420 : "1.4.0", 1421 # combination version pattern 1422 ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "", "") 1423 : "1.2.0", 1424 ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "", "") 1425 : "2014.01", 1426 # version pattern "yyyymmdd" 1427 ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "", "") 1428 : "20120614", 1429 # packages with a valid UPSTREAM_CHECK_GITTAGREGEX 1430 # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing 1431 ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap;branch=master;protocol=https", "ae0394e687f1a77e966cf72f895da91840dffb8f", r"(?P<pver>(\d+\.(\d\.?)*))", "") 1432 : "0.4.3", 1433 ("build-appliance-image", "git://git.yoctoproject.org/poky;branch=master;protocol=https", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P<pver>(([0-9][\.|_]?)+[0-9]))", "") 1434 : "11.0.0", 1435 ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))", "") 1436 : "1.3.59", 1437 ("remake", "git://github.com/rocky/remake.git;protocol=https;branch=master", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))", "") 1438 : "3.82+dbg0.9", 1439 ("sysdig", "git://github.com/draios/sysdig.git;branch=dev;protocol=https", "4fb6288275f567f63515df0ff0a6518043ecfa9b", r"^(?P<pver>\d+(\.\d+)+)", "10.0.0") 1440 : "0.28.0", 1441 } 1442 1443 test_wget_uris = { 1444 # 1445 # packages with versions inside directory name 1446 # 1447 # http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2 1448 ("util-linux", "/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2", "", "") 1449 : "2.24.2", 1450 # http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz 1451 ("enchant", "/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz", "", "") 1452 : "1.6.0", 1453 # http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz 1454 ("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz", "", "") 1455 : "2.8.12.1", 1456 # https://download.gnome.org/sources/libxml2/2.9/libxml2-2.9.14.tar.xz 1457 ("libxml2", "/software/libxml2/2.9/libxml2-2.9.14.tar.xz", "", "") 1458 : "2.10.3", 1459 # 1460 # packages with versions only in current directory 1461 # 1462 # https://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2 1463 ("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "") 1464 : "2.19", 1465 # https://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2 1466 ("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "") 1467 : "20120814", 1468 # 1469 # packages with "99" in the name of possible version 1470 # 1471 # http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz 1472 ("pulseaudio", "/software/pulseaudio/releases/pulseaudio-4.0.tar.xz", "", "") 1473 : "5.0", 1474 # http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2 1475 ("xserver-xorg", "/releases/individual/xserver/xorg-server-1.15.1.tar.bz2", "", "") 1476 : "1.15.1", 1477 # 1478 # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX 1479 # 1480 # http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2 1481 # https://github.com/apple/cups/releases 1482 ("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", "/apple/cups/releases", r"(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz") 1483 : "2.0.0", 1484 # http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz 1485 # http://ftp.debian.org/debian/pool/main/d/db5.3/ 1486 ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", r"(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz") 1487 : "5.3.10", 1488 # 1489 # packages where the tarball compression changed in the new version 1490 # 1491 # http://ftp.debian.org/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz 1492 ("minicom", "/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz", "", "") 1493 : "2.8", 1494 } 1495 1496 @skipIfNoNetwork() 1497 def test_git_latest_versionstring(self): 1498 for k, v in self.test_git_uris.items(): 1499 self.d.setVar("PN", k[0]) 1500 self.d.setVar("SRCREV", k[2]) 1501 self.d.setVar("UPSTREAM_CHECK_GITTAGREGEX", k[3]) 1502 ud = bb.fetch2.FetchData(k[1], self.d) 1503 pupver= ud.method.latest_versionstring(ud, self.d) 1504 verstring = pupver[0] 1505 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) 1506 r = bb.utils.vercmp_string(v, verstring) 1507 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) 1508 if k[4]: 1509 r = bb.utils.vercmp_string(verstring, k[4]) 1510 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], verstring, k[4])) 1511 1512 def test_wget_latest_versionstring(self): 1513 testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata" 1514 server = HTTPService(testdata) 1515 server.start() 1516 port = server.port 1517 try: 1518 for k, v in self.test_wget_uris.items(): 1519 self.d.setVar("PN", k[0]) 1520 checkuri = "" 1521 if k[2]: 1522 checkuri = "http://localhost:%s/" % port + k[2] 1523 self.d.setVar("UPSTREAM_CHECK_URI", checkuri) 1524 self.d.setVar("UPSTREAM_CHECK_REGEX", k[3]) 1525 url = "http://localhost:%s/" % port + k[1] 1526 ud = bb.fetch2.FetchData(url, self.d) 1527 pupver = ud.method.latest_versionstring(ud, self.d) 1528 verstring = pupver[0] 1529 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) 1530 r = bb.utils.vercmp_string(v, verstring) 1531 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) 1532 finally: 1533 server.stop() 1534 1535 1536class FetchCheckStatusTest(FetcherTest): 1537 test_wget_uris = ["https://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz", 1538 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz", 1539 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz", 1540 "https://yoctoproject.org/", 1541 "https://docs.yoctoproject.org", 1542 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz", 1543 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz", 1544 "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz", 1545 # GitHub releases are hosted on Amazon S3, which doesn't support HEAD 1546 "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz" 1547 ] 1548 1549 @skipIfNoNetwork() 1550 def test_wget_checkstatus(self): 1551 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d) 1552 for u in self.test_wget_uris: 1553 with self.subTest(url=u): 1554 ud = fetch.ud[u] 1555 m = ud.method 1556 ret = m.checkstatus(fetch, ud, self.d) 1557 self.assertTrue(ret, msg="URI %s, can't check status" % (u)) 1558 1559 @skipIfNoNetwork() 1560 def test_wget_checkstatus_connection_cache(self): 1561 from bb.fetch2 import FetchConnectionCache 1562 1563 connection_cache = FetchConnectionCache() 1564 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d, 1565 connection_cache = connection_cache) 1566 1567 for u in self.test_wget_uris: 1568 with self.subTest(url=u): 1569 ud = fetch.ud[u] 1570 m = ud.method 1571 ret = m.checkstatus(fetch, ud, self.d) 1572 self.assertTrue(ret, msg="URI %s, can't check status" % (u)) 1573 1574 connection_cache.close_connections() 1575 1576 1577class GitMakeShallowTest(FetcherTest): 1578 def setUp(self): 1579 FetcherTest.setUp(self) 1580 self.gitdir = os.path.join(self.tempdir, 'gitshallow') 1581 bb.utils.mkdirhier(self.gitdir) 1582 self.git_init() 1583 1584 def assertRefs(self, expected_refs): 1585 actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines() 1586 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs).splitlines() 1587 self.assertEqual(sorted(full_expected), sorted(actual_refs)) 1588 1589 def assertRevCount(self, expected_count, args=None): 1590 if args is None: 1591 args = ['HEAD'] 1592 revs = self.git(['rev-list'] + args) 1593 actual_count = len(revs.splitlines()) 1594 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count)) 1595 1596 def make_shallow(self, args=None): 1597 if args is None: 1598 args = ['HEAD'] 1599 return bb.process.run([bb.fetch2.git.Git.make_shallow_path] + args, cwd=self.gitdir) 1600 1601 def add_empty_file(self, path, msg=None): 1602 if msg is None: 1603 msg = path 1604 open(os.path.join(self.gitdir, path), 'w').close() 1605 self.git(['add', path]) 1606 self.git(['commit', '-m', msg, path]) 1607 1608 def test_make_shallow_single_branch_no_merge(self): 1609 self.add_empty_file('a') 1610 self.add_empty_file('b') 1611 self.assertRevCount(2) 1612 self.make_shallow() 1613 self.assertRevCount(1) 1614 1615 def test_make_shallow_single_branch_one_merge(self): 1616 self.add_empty_file('a') 1617 self.add_empty_file('b') 1618 self.git('checkout -b a_branch') 1619 self.add_empty_file('c') 1620 self.git('checkout master') 1621 self.add_empty_file('d') 1622 self.git('merge --no-ff --no-edit a_branch') 1623 self.git('branch -d a_branch') 1624 self.add_empty_file('e') 1625 self.assertRevCount(6) 1626 self.make_shallow(['HEAD~2']) 1627 self.assertRevCount(5) 1628 1629 def test_make_shallow_at_merge(self): 1630 self.add_empty_file('a') 1631 self.git('checkout -b a_branch') 1632 self.add_empty_file('b') 1633 self.git('checkout master') 1634 self.git('merge --no-ff --no-edit a_branch') 1635 self.git('branch -d a_branch') 1636 self.assertRevCount(3) 1637 self.make_shallow() 1638 self.assertRevCount(1) 1639 1640 def test_make_shallow_annotated_tag(self): 1641 self.add_empty_file('a') 1642 self.add_empty_file('b') 1643 self.git('tag -a -m a_tag a_tag') 1644 self.assertRevCount(2) 1645 self.make_shallow(['a_tag']) 1646 self.assertRevCount(1) 1647 1648 def test_make_shallow_multi_ref(self): 1649 self.add_empty_file('a') 1650 self.add_empty_file('b') 1651 self.git('checkout -b a_branch') 1652 self.add_empty_file('c') 1653 self.git('checkout master') 1654 self.add_empty_file('d') 1655 self.git('checkout -b a_branch_2') 1656 self.add_empty_file('a_tag') 1657 self.git('tag a_tag') 1658 self.git('checkout master') 1659 self.git('branch -D a_branch_2') 1660 self.add_empty_file('e') 1661 self.assertRevCount(6, ['--all']) 1662 self.make_shallow() 1663 self.assertRevCount(5, ['--all']) 1664 1665 def test_make_shallow_multi_ref_trim(self): 1666 self.add_empty_file('a') 1667 self.git('checkout -b a_branch') 1668 self.add_empty_file('c') 1669 self.git('checkout master') 1670 self.assertRevCount(1) 1671 self.assertRevCount(2, ['--all']) 1672 self.assertRefs(['master', 'a_branch']) 1673 self.make_shallow(['-r', 'master', 'HEAD']) 1674 self.assertRevCount(1, ['--all']) 1675 self.assertRefs(['master']) 1676 1677 def test_make_shallow_noop(self): 1678 self.add_empty_file('a') 1679 self.assertRevCount(1) 1680 self.make_shallow() 1681 self.assertRevCount(1) 1682 1683 @skipIfNoNetwork() 1684 def test_make_shallow_bitbake(self): 1685 self.git('remote add origin https://github.com/openembedded/bitbake') 1686 self.git('fetch --tags origin') 1687 orig_revs = len(self.git('rev-list --all').splitlines()) 1688 self.make_shallow(['refs/tags/1.10.0']) 1689 self.assertRevCount(orig_revs - 1746, ['--all']) 1690 1691class GitShallowTest(FetcherTest): 1692 def setUp(self): 1693 FetcherTest.setUp(self) 1694 self.gitdir = os.path.join(self.tempdir, 'git') 1695 self.srcdir = os.path.join(self.tempdir, 'gitsource') 1696 1697 bb.utils.mkdirhier(self.srcdir) 1698 self.git_init(cwd=self.srcdir) 1699 self.d.setVar('WORKDIR', self.tempdir) 1700 self.d.setVar('S', self.gitdir) 1701 self.d.delVar('PREMIRRORS') 1702 self.d.delVar('MIRRORS') 1703 1704 uri = 'git://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 1705 self.d.setVar('SRC_URI', uri) 1706 self.d.setVar('SRCREV', '${AUTOREV}') 1707 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') 1708 1709 self.d.setVar('BB_GIT_SHALLOW', '1') 1710 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0') 1711 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 1712 self.d.setVar("__BBSRCREV_SEEN", "1") 1713 1714 def assertRefs(self, expected_refs, cwd=None): 1715 if cwd is None: 1716 cwd = self.gitdir 1717 actual_refs = self.git(['for-each-ref', '--format=%(refname)'], cwd=cwd).splitlines() 1718 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs, cwd=cwd).splitlines() 1719 self.assertEqual(sorted(set(full_expected)), sorted(set(actual_refs))) 1720 1721 def assertRevCount(self, expected_count, args=None, cwd=None): 1722 if args is None: 1723 args = ['HEAD'] 1724 if cwd is None: 1725 cwd = self.gitdir 1726 revs = self.git(['rev-list'] + args, cwd=cwd) 1727 actual_count = len(revs.splitlines()) 1728 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count)) 1729 1730 def add_empty_file(self, path, cwd=None, msg=None): 1731 if msg is None: 1732 msg = path 1733 if cwd is None: 1734 cwd = self.srcdir 1735 open(os.path.join(cwd, path), 'w').close() 1736 self.git(['add', path], cwd) 1737 self.git(['commit', '-m', msg, path], cwd) 1738 1739 def fetch(self, uri=None): 1740 if uri is None: 1741 uris = self.d.getVar('SRC_URI').split() 1742 uri = uris[0] 1743 d = self.d 1744 else: 1745 d = self.d.createCopy() 1746 d.setVar('SRC_URI', uri) 1747 uri = d.expand(uri) 1748 uris = [uri] 1749 1750 fetcher = bb.fetch2.Fetch(uris, d) 1751 fetcher.download() 1752 ud = fetcher.ud[uri] 1753 return fetcher, ud 1754 1755 def fetch_and_unpack(self, uri=None): 1756 fetcher, ud = self.fetch(uri) 1757 fetcher.unpack(self.d.getVar('WORKDIR')) 1758 assert os.path.exists(self.d.getVar('S')) 1759 return fetcher, ud 1760 1761 def fetch_shallow(self, uri=None, disabled=False, keepclone=False): 1762 """Fetch a uri, generating a shallow tarball, then unpack using it""" 1763 fetcher, ud = self.fetch_and_unpack(uri) 1764 assert os.path.exists(ud.clonedir), 'Git clone in DLDIR (%s) does not exist for uri %s' % (ud.clonedir, uri) 1765 1766 # Confirm that the unpacked repo is unshallow 1767 if not disabled: 1768 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0])) 1769 1770 # fetch and unpack, from the shallow tarball 1771 bb.utils.remove(self.gitdir, recurse=True) 1772 bb.process.run('chmod u+w -R "%s"' % ud.clonedir) 1773 bb.utils.remove(ud.clonedir, recurse=True) 1774 bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True) 1775 1776 # confirm that the unpacked repo is used when no git clone or git 1777 # mirror tarball is available 1778 fetcher, ud = self.fetch_and_unpack(uri) 1779 if not disabled: 1780 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is not shallow' % self.gitdir 1781 else: 1782 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is shallow' % self.gitdir 1783 return fetcher, ud 1784 1785 def test_shallow_disabled(self): 1786 self.add_empty_file('a') 1787 self.add_empty_file('b') 1788 self.assertRevCount(2, cwd=self.srcdir) 1789 1790 self.d.setVar('BB_GIT_SHALLOW', '0') 1791 self.fetch_shallow(disabled=True) 1792 self.assertRevCount(2) 1793 1794 def test_shallow_nobranch(self): 1795 self.add_empty_file('a') 1796 self.add_empty_file('b') 1797 self.assertRevCount(2, cwd=self.srcdir) 1798 1799 srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip() 1800 self.d.setVar('SRCREV', srcrev) 1801 uri = self.d.getVar('SRC_URI').split()[0] 1802 uri = '%s;nobranch=1;bare=1' % uri 1803 1804 self.fetch_shallow(uri) 1805 self.assertRevCount(1) 1806 1807 # shallow refs are used to ensure the srcrev sticks around when we 1808 # have no other branches referencing it 1809 self.assertRefs(['refs/shallow/default']) 1810 1811 def test_shallow_default_depth_1(self): 1812 # Create initial git repo 1813 self.add_empty_file('a') 1814 self.add_empty_file('b') 1815 self.assertRevCount(2, cwd=self.srcdir) 1816 1817 self.fetch_shallow() 1818 self.assertRevCount(1) 1819 1820 def test_shallow_depth_0_disables(self): 1821 self.add_empty_file('a') 1822 self.add_empty_file('b') 1823 self.assertRevCount(2, cwd=self.srcdir) 1824 1825 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 1826 self.fetch_shallow(disabled=True) 1827 self.assertRevCount(2) 1828 1829 def test_shallow_depth_default_override(self): 1830 self.add_empty_file('a') 1831 self.add_empty_file('b') 1832 self.assertRevCount(2, cwd=self.srcdir) 1833 1834 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '2') 1835 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '1') 1836 self.fetch_shallow() 1837 self.assertRevCount(1) 1838 1839 def test_shallow_depth_default_override_disable(self): 1840 self.add_empty_file('a') 1841 self.add_empty_file('b') 1842 self.add_empty_file('c') 1843 self.assertRevCount(3, cwd=self.srcdir) 1844 1845 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 1846 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '2') 1847 self.fetch_shallow() 1848 self.assertRevCount(2) 1849 1850 def test_current_shallow_out_of_date_clone(self): 1851 # Create initial git repo 1852 self.add_empty_file('a') 1853 self.add_empty_file('b') 1854 self.add_empty_file('c') 1855 self.assertRevCount(3, cwd=self.srcdir) 1856 1857 # Clone and generate mirror tarball 1858 fetcher, ud = self.fetch() 1859 1860 # Ensure we have a current mirror tarball, but an out of date clone 1861 self.git('update-ref refs/heads/master refs/heads/master~1', cwd=ud.clonedir) 1862 self.assertRevCount(2, cwd=ud.clonedir) 1863 1864 # Fetch and unpack, from the current tarball, not the out of date clone 1865 bb.utils.remove(self.gitdir, recurse=True) 1866 fetcher, ud = self.fetch() 1867 fetcher.unpack(self.d.getVar('WORKDIR')) 1868 self.assertRevCount(1) 1869 1870 def test_shallow_single_branch_no_merge(self): 1871 self.add_empty_file('a') 1872 self.add_empty_file('b') 1873 self.assertRevCount(2, cwd=self.srcdir) 1874 1875 self.fetch_shallow() 1876 self.assertRevCount(1) 1877 assert os.path.exists(os.path.join(self.gitdir, 'a')) 1878 assert os.path.exists(os.path.join(self.gitdir, 'b')) 1879 1880 def test_shallow_no_dangling(self): 1881 self.add_empty_file('a') 1882 self.add_empty_file('b') 1883 self.assertRevCount(2, cwd=self.srcdir) 1884 1885 self.fetch_shallow() 1886 self.assertRevCount(1) 1887 assert not self.git('fsck --dangling') 1888 1889 def test_shallow_srcrev_branch_truncation(self): 1890 self.add_empty_file('a') 1891 self.add_empty_file('b') 1892 b_commit = self.git('rev-parse HEAD', cwd=self.srcdir).rstrip() 1893 self.add_empty_file('c') 1894 self.assertRevCount(3, cwd=self.srcdir) 1895 1896 self.d.setVar('SRCREV', b_commit) 1897 self.fetch_shallow() 1898 1899 # The 'c' commit was removed entirely, and 'a' was removed from history 1900 self.assertRevCount(1, ['--all']) 1901 self.assertEqual(self.git('rev-parse HEAD').strip(), b_commit) 1902 assert os.path.exists(os.path.join(self.gitdir, 'a')) 1903 assert os.path.exists(os.path.join(self.gitdir, 'b')) 1904 assert not os.path.exists(os.path.join(self.gitdir, 'c')) 1905 1906 def test_shallow_ref_pruning(self): 1907 self.add_empty_file('a') 1908 self.add_empty_file('b') 1909 self.git('branch a_branch', cwd=self.srcdir) 1910 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir) 1911 self.assertRevCount(2, cwd=self.srcdir) 1912 1913 self.fetch_shallow() 1914 1915 self.assertRefs(['master', 'origin/master']) 1916 self.assertRevCount(1) 1917 1918 def test_shallow_submodules(self): 1919 self.add_empty_file('a') 1920 self.add_empty_file('b') 1921 1922 smdir = os.path.join(self.tempdir, 'gitsubmodule') 1923 bb.utils.mkdirhier(smdir) 1924 self.git_init(cwd=smdir) 1925 # Make this look like it was cloned from a remote... 1926 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) 1927 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) 1928 self.add_empty_file('asub', cwd=smdir) 1929 self.add_empty_file('bsub', cwd=smdir) 1930 1931 self.git('submodule init', cwd=self.srcdir) 1932 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir) 1933 self.git('submodule update', cwd=self.srcdir) 1934 self.git('commit -m submodule -a', cwd=self.srcdir) 1935 1936 uri = 'gitsm://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 1937 fetcher, ud = self.fetch_shallow(uri) 1938 1939 # Verify the main repository is shallow 1940 self.assertRevCount(1) 1941 1942 # Verify the gitsubmodule directory is present 1943 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule')) 1944 1945 # Verify the submodule is also shallow 1946 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule')) 1947 1948 def test_shallow_submodule_mirrors(self): 1949 self.add_empty_file('a') 1950 self.add_empty_file('b') 1951 1952 smdir = os.path.join(self.tempdir, 'gitsubmodule') 1953 bb.utils.mkdirhier(smdir) 1954 self.git_init(cwd=smdir) 1955 # Make this look like it was cloned from a remote... 1956 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) 1957 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) 1958 self.add_empty_file('asub', cwd=smdir) 1959 self.add_empty_file('bsub', cwd=smdir) 1960 1961 self.git('submodule init', cwd=self.srcdir) 1962 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir) 1963 self.git('submodule update', cwd=self.srcdir) 1964 self.git('commit -m submodule -a', cwd=self.srcdir) 1965 1966 uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir 1967 1968 # Fetch once to generate the shallow tarball 1969 fetcher, ud = self.fetch(uri) 1970 1971 # Set up the mirror 1972 mirrordir = os.path.join(self.tempdir, 'mirror') 1973 bb.utils.rename(self.dldir, mirrordir) 1974 self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/' % mirrordir) 1975 1976 # Fetch from the mirror 1977 bb.utils.remove(self.dldir, recurse=True) 1978 bb.utils.remove(self.gitdir, recurse=True) 1979 self.fetch_and_unpack(uri) 1980 1981 # Verify the main repository is shallow 1982 self.assertRevCount(1) 1983 1984 # Verify the gitsubmodule directory is present 1985 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule')) 1986 1987 # Verify the submodule is also shallow 1988 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule')) 1989 1990 if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')): 1991 def test_shallow_annex(self): 1992 self.add_empty_file('a') 1993 self.add_empty_file('b') 1994 self.git('annex init', cwd=self.srcdir) 1995 open(os.path.join(self.srcdir, 'c'), 'w').close() 1996 self.git('annex add c', cwd=self.srcdir) 1997 self.git('commit --author "Foo Bar <foo@bar>" -m annex-c -a', cwd=self.srcdir) 1998 bb.process.run('chmod u+w -R %s' % self.srcdir) 1999 2000 uri = 'gitannex://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 2001 fetcher, ud = self.fetch_shallow(uri) 2002 2003 self.assertRevCount(1) 2004 assert './.git/annex/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0] 2005 assert os.path.exists(os.path.join(self.gitdir, 'c')) 2006 2007 def test_shallow_multi_one_uri(self): 2008 # Create initial git repo 2009 self.add_empty_file('a') 2010 self.add_empty_file('b') 2011 self.git('checkout -b a_branch', cwd=self.srcdir) 2012 self.add_empty_file('c') 2013 self.add_empty_file('d') 2014 self.git('checkout master', cwd=self.srcdir) 2015 self.git('tag v0.0 a_branch', cwd=self.srcdir) 2016 self.add_empty_file('e') 2017 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 2018 self.add_empty_file('f') 2019 self.assertRevCount(7, cwd=self.srcdir) 2020 2021 uri = self.d.getVar('SRC_URI').split()[0] 2022 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri 2023 2024 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2025 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2026 self.d.setVar('SRCREV_master', '${AUTOREV}') 2027 self.d.setVar('SRCREV_a_branch', '${AUTOREV}') 2028 2029 self.fetch_shallow(uri) 2030 2031 self.assertRevCount(5) 2032 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 2033 2034 def test_shallow_multi_one_uri_depths(self): 2035 # Create initial git repo 2036 self.add_empty_file('a') 2037 self.add_empty_file('b') 2038 self.git('checkout -b a_branch', cwd=self.srcdir) 2039 self.add_empty_file('c') 2040 self.add_empty_file('d') 2041 self.git('checkout master', cwd=self.srcdir) 2042 self.add_empty_file('e') 2043 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 2044 self.add_empty_file('f') 2045 self.assertRevCount(7, cwd=self.srcdir) 2046 2047 uri = self.d.getVar('SRC_URI').split()[0] 2048 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri 2049 2050 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2051 self.d.setVar('BB_GIT_SHALLOW_DEPTH_master', '3') 2052 self.d.setVar('BB_GIT_SHALLOW_DEPTH_a_branch', '1') 2053 self.d.setVar('SRCREV_master', '${AUTOREV}') 2054 self.d.setVar('SRCREV_a_branch', '${AUTOREV}') 2055 2056 self.fetch_shallow(uri) 2057 2058 self.assertRevCount(4, ['--all']) 2059 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 2060 2061 def test_shallow_clone_preferred_over_shallow(self): 2062 self.add_empty_file('a') 2063 self.add_empty_file('b') 2064 2065 # Fetch once to generate the shallow tarball 2066 fetcher, ud = self.fetch() 2067 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0])) 2068 2069 # Fetch and unpack with both the clonedir and shallow tarball available 2070 bb.utils.remove(self.gitdir, recurse=True) 2071 fetcher, ud = self.fetch_and_unpack() 2072 2073 # The unpacked tree should *not* be shallow 2074 self.assertRevCount(2) 2075 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')) 2076 2077 def test_shallow_mirrors(self): 2078 self.add_empty_file('a') 2079 self.add_empty_file('b') 2080 2081 # Fetch once to generate the shallow tarball 2082 fetcher, ud = self.fetch() 2083 mirrortarball = ud.mirrortarballs[0] 2084 assert os.path.exists(os.path.join(self.dldir, mirrortarball)) 2085 2086 # Set up the mirror 2087 mirrordir = os.path.join(self.tempdir, 'mirror') 2088 bb.utils.mkdirhier(mirrordir) 2089 self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/' % mirrordir) 2090 2091 bb.utils.rename(os.path.join(self.dldir, mirrortarball), 2092 os.path.join(mirrordir, mirrortarball)) 2093 2094 # Fetch from the mirror 2095 bb.utils.remove(self.dldir, recurse=True) 2096 bb.utils.remove(self.gitdir, recurse=True) 2097 self.fetch_and_unpack() 2098 self.assertRevCount(1) 2099 2100 def test_shallow_invalid_depth(self): 2101 self.add_empty_file('a') 2102 self.add_empty_file('b') 2103 2104 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '-12') 2105 with self.assertRaises(bb.fetch2.FetchError): 2106 self.fetch() 2107 2108 def test_shallow_invalid_depth_default(self): 2109 self.add_empty_file('a') 2110 self.add_empty_file('b') 2111 2112 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '-12') 2113 with self.assertRaises(bb.fetch2.FetchError): 2114 self.fetch() 2115 2116 def test_shallow_extra_refs(self): 2117 self.add_empty_file('a') 2118 self.add_empty_file('b') 2119 self.git('branch a_branch', cwd=self.srcdir) 2120 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir) 2121 self.assertRevCount(2, cwd=self.srcdir) 2122 2123 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/a_branch') 2124 self.fetch_shallow() 2125 2126 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 2127 self.assertRevCount(1) 2128 2129 def test_shallow_extra_refs_wildcard(self): 2130 self.add_empty_file('a') 2131 self.add_empty_file('b') 2132 self.git('branch a_branch', cwd=self.srcdir) 2133 self.git('tag v1.0', cwd=self.srcdir) 2134 self.assertRefs(['master', 'a_branch', 'v1.0'], cwd=self.srcdir) 2135 self.assertRevCount(2, cwd=self.srcdir) 2136 2137 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*') 2138 self.fetch_shallow() 2139 2140 self.assertRefs(['master', 'origin/master', 'v1.0']) 2141 self.assertRevCount(1) 2142 2143 def test_shallow_missing_extra_refs(self): 2144 self.add_empty_file('a') 2145 self.add_empty_file('b') 2146 2147 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/foo') 2148 with self.assertRaises(bb.fetch2.FetchError): 2149 self.fetch() 2150 2151 def test_shallow_missing_extra_refs_wildcard(self): 2152 self.add_empty_file('a') 2153 self.add_empty_file('b') 2154 2155 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*') 2156 self.fetch() 2157 2158 def test_shallow_remove_revs(self): 2159 # Create initial git repo 2160 self.add_empty_file('a') 2161 self.add_empty_file('b') 2162 self.git('checkout -b a_branch', cwd=self.srcdir) 2163 self.add_empty_file('c') 2164 self.add_empty_file('d') 2165 self.git('checkout master', cwd=self.srcdir) 2166 self.git('tag v0.0 a_branch', cwd=self.srcdir) 2167 self.add_empty_file('e') 2168 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 2169 self.git('branch -d a_branch', cwd=self.srcdir) 2170 self.add_empty_file('f') 2171 self.assertRevCount(7, cwd=self.srcdir) 2172 2173 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2174 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2175 2176 self.fetch_shallow() 2177 2178 self.assertRevCount(5) 2179 2180 def test_shallow_invalid_revs(self): 2181 self.add_empty_file('a') 2182 self.add_empty_file('b') 2183 2184 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2185 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2186 2187 with self.assertRaises(bb.fetch2.FetchError): 2188 self.fetch() 2189 2190 def test_shallow_fetch_missing_revs(self): 2191 self.add_empty_file('a') 2192 self.add_empty_file('b') 2193 fetcher, ud = self.fetch(self.d.getVar('SRC_URI')) 2194 self.git('tag v0.0 master', cwd=self.srcdir) 2195 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2196 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2197 self.fetch_shallow() 2198 2199 def test_shallow_fetch_missing_revs_fails(self): 2200 self.add_empty_file('a') 2201 self.add_empty_file('b') 2202 fetcher, ud = self.fetch(self.d.getVar('SRC_URI')) 2203 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2204 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2205 2206 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm: 2207 self.fetch_shallow() 2208 self.assertIn("Unable to find revision v0.0 even from upstream", cm.output[0]) 2209 2210 @skipIfNoNetwork() 2211 def test_bitbake(self): 2212 self.git('remote add --mirror=fetch origin https://github.com/openembedded/bitbake', cwd=self.srcdir) 2213 self.git('config core.bare true', cwd=self.srcdir) 2214 self.git('fetch', cwd=self.srcdir) 2215 2216 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2217 # Note that the 1.10.0 tag is annotated, so this also tests 2218 # reference of an annotated vs unannotated tag 2219 self.d.setVar('BB_GIT_SHALLOW_REVS', '1.10.0') 2220 2221 self.fetch_shallow() 2222 2223 # Confirm that the history of 1.10.0 was removed 2224 orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines()) 2225 revs = len(self.git('rev-list master').splitlines()) 2226 self.assertNotEqual(orig_revs, revs) 2227 self.assertRefs(['master', 'origin/master']) 2228 self.assertRevCount(orig_revs - 1758) 2229 2230 def test_that_unpack_throws_an_error_when_the_git_clone_nor_shallow_tarball_exist(self): 2231 self.add_empty_file('a') 2232 fetcher, ud = self.fetch() 2233 bb.utils.remove(self.gitdir, recurse=True) 2234 bb.utils.remove(self.dldir, recurse=True) 2235 2236 with self.assertRaises(bb.fetch2.UnpackError) as context: 2237 fetcher.unpack(self.d.getVar('WORKDIR')) 2238 2239 self.assertIn("No up to date source found", context.exception.msg) 2240 self.assertIn("clone directory not available or not up to date", context.exception.msg) 2241 2242 @skipIfNoNetwork() 2243 def test_that_unpack_does_work_when_using_git_shallow_tarball_but_tarball_is_not_available(self): 2244 self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0') 2245 self.d.setVar('BB_GIT_SHALLOW', '1') 2246 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 2247 fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests;branch=master;protocol=https"], self.d) 2248 fetcher.download() 2249 2250 bb.utils.remove(self.dldir + "/*.tar.gz") 2251 fetcher.unpack(self.unpackdir) 2252 2253 dir = os.listdir(self.unpackdir + "/git/") 2254 self.assertIn("fstests.doap", dir) 2255 2256class GitLfsTest(FetcherTest): 2257 def skipIfNoGitLFS(): 2258 import shutil 2259 if not shutil.which('git-lfs'): 2260 return unittest.skip('git-lfs not installed') 2261 return lambda f: f 2262 2263 def setUp(self): 2264 FetcherTest.setUp(self) 2265 2266 self.gitdir = os.path.join(self.tempdir, 'git') 2267 self.srcdir = os.path.join(self.tempdir, 'gitsource') 2268 2269 self.d.setVar('WORKDIR', self.tempdir) 2270 self.d.setVar('S', self.gitdir) 2271 self.d.delVar('PREMIRRORS') 2272 self.d.delVar('MIRRORS') 2273 2274 self.d.setVar('SRCREV', '${AUTOREV}') 2275 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') 2276 self.d.setVar("__BBSRCREV_SEEN", "1") 2277 2278 bb.utils.mkdirhier(self.srcdir) 2279 self.git_init(cwd=self.srcdir) 2280 self.commit_file('.gitattributes', '*.mp3 filter=lfs -text') 2281 2282 def commit_file(self, filename, content): 2283 with open(os.path.join(self.srcdir, filename), "w") as f: 2284 f.write(content) 2285 self.git(["add", filename], cwd=self.srcdir) 2286 self.git(["commit", "-m", "Change"], cwd=self.srcdir) 2287 return self.git(["rev-parse", "HEAD"], cwd=self.srcdir).strip() 2288 2289 def fetch(self, uri=None, download=True): 2290 uris = self.d.getVar('SRC_URI').split() 2291 uri = uris[0] 2292 d = self.d 2293 2294 fetcher = bb.fetch2.Fetch(uris, d) 2295 if download: 2296 fetcher.download() 2297 ud = fetcher.ud[uri] 2298 return fetcher, ud 2299 2300 def get_real_git_lfs_file(self): 2301 self.d.setVar('PATH', os.environ.get('PATH')) 2302 fetcher, ud = self.fetch() 2303 fetcher.unpack(self.d.getVar('WORKDIR')) 2304 unpacked_lfs_file = os.path.join(self.d.getVar('WORKDIR'), 'git', "Cat_poster_1.jpg") 2305 return unpacked_lfs_file 2306 2307 @skipIfNoGitLFS() 2308 def test_fetch_lfs_on_srcrev_change(self): 2309 """Test if fetch downloads missing LFS objects when a different revision within an existing repository is requested""" 2310 self.git(["lfs", "install", "--local"], cwd=self.srcdir) 2311 2312 @contextlib.contextmanager 2313 def hide_upstream_repository(): 2314 """Hide the upstream repository to make sure that git lfs cannot pull from it""" 2315 temp_name = self.srcdir + ".bak" 2316 os.rename(self.srcdir, temp_name) 2317 try: 2318 yield 2319 finally: 2320 os.rename(temp_name, self.srcdir) 2321 2322 def fetch_and_verify(revision, filename, content): 2323 self.d.setVar('SRCREV', revision) 2324 fetcher, ud = self.fetch() 2325 2326 with hide_upstream_repository(): 2327 workdir = self.d.getVar('WORKDIR') 2328 fetcher.unpack(workdir) 2329 2330 with open(os.path.join(workdir, "git", filename)) as f: 2331 self.assertEqual(f.read(), content) 2332 2333 commit_1 = self.commit_file("a.mp3", "version 1") 2334 commit_2 = self.commit_file("a.mp3", "version 2") 2335 2336 self.d.setVar('SRC_URI', "git://%s;protocol=file;lfs=1;branch=master" % self.srcdir) 2337 2338 # Seed the local download folder by fetching the latest commit and verifying that the LFS contents are 2339 # available even when the upstream repository disappears. 2340 fetch_and_verify(commit_2, "a.mp3", "version 2") 2341 # Verify that even when an older revision is fetched, the needed LFS objects are fetched into the download 2342 # folder. 2343 fetch_and_verify(commit_1, "a.mp3", "version 1") 2344 2345 @skipIfNoGitLFS() 2346 @skipIfNoNetwork() 2347 def test_real_git_lfs_repo_succeeds_without_lfs_param(self): 2348 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master") 2349 f = self.get_real_git_lfs_file() 2350 self.assertTrue(os.path.exists(f)) 2351 self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f)) 2352 2353 @skipIfNoGitLFS() 2354 @skipIfNoNetwork() 2355 def test_real_git_lfs_repo_succeeds(self): 2356 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=1") 2357 f = self.get_real_git_lfs_file() 2358 self.assertTrue(os.path.exists(f)) 2359 self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f)) 2360 2361 @skipIfNoGitLFS() 2362 @skipIfNoNetwork() 2363 def test_real_git_lfs_repo_skips(self): 2364 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=0") 2365 f = self.get_real_git_lfs_file() 2366 # This is the actual non-smudged placeholder file on the repo if git-lfs does not run 2367 lfs_file = ( 2368 'version https://git-lfs.github.com/spec/v1\n' 2369 'oid sha256:34be66b1a39a1955b46a12588df9d5f6fc1da790e05cf01f3c7422f4bbbdc26b\n' 2370 'size 11423554\n' 2371 ) 2372 2373 with open(f) as fh: 2374 self.assertEqual(lfs_file, fh.read()) 2375 2376 @skipIfNoGitLFS() 2377 def test_lfs_enabled(self): 2378 import shutil 2379 2380 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir 2381 self.d.setVar('SRC_URI', uri) 2382 2383 # With git-lfs installed, test that we can fetch and unpack 2384 fetcher, ud = self.fetch() 2385 shutil.rmtree(self.gitdir, ignore_errors=True) 2386 fetcher.unpack(self.d.getVar('WORKDIR')) 2387 2388 @skipIfNoGitLFS() 2389 def test_lfs_disabled(self): 2390 import shutil 2391 2392 uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir 2393 self.d.setVar('SRC_URI', uri) 2394 2395 # Verify that the fetcher can survive even if the source 2396 # repository has Git LFS usage configured. 2397 fetcher, ud = self.fetch() 2398 fetcher.unpack(self.d.getVar('WORKDIR')) 2399 2400 def test_lfs_enabled_not_installed(self): 2401 import shutil 2402 2403 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir 2404 self.d.setVar('SRC_URI', uri) 2405 2406 # Careful: suppress initial attempt at downloading 2407 fetcher, ud = self.fetch(uri=None, download=False) 2408 2409 # Artificially assert that git-lfs is not installed, so 2410 # we can verify a failure to unpack in it's absence. 2411 old_find_git_lfs = ud.method._find_git_lfs 2412 try: 2413 # If git-lfs cannot be found, the unpack should throw an error 2414 with self.assertRaises(bb.fetch2.FetchError): 2415 fetcher.download() 2416 ud.method._find_git_lfs = lambda d: False 2417 shutil.rmtree(self.gitdir, ignore_errors=True) 2418 fetcher.unpack(self.d.getVar('WORKDIR')) 2419 finally: 2420 ud.method._find_git_lfs = old_find_git_lfs 2421 2422 def test_lfs_disabled_not_installed(self): 2423 import shutil 2424 2425 uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir 2426 self.d.setVar('SRC_URI', uri) 2427 2428 # Careful: suppress initial attempt at downloading 2429 fetcher, ud = self.fetch(uri=None, download=False) 2430 2431 # Artificially assert that git-lfs is not installed, so 2432 # we can verify a failure to unpack in it's absence. 2433 old_find_git_lfs = ud.method._find_git_lfs 2434 try: 2435 # Even if git-lfs cannot be found, the unpack should be successful 2436 fetcher.download() 2437 ud.method._find_git_lfs = lambda d: False 2438 shutil.rmtree(self.gitdir, ignore_errors=True) 2439 fetcher.unpack(self.d.getVar('WORKDIR')) 2440 finally: 2441 ud.method._find_git_lfs = old_find_git_lfs 2442 2443class GitURLWithSpacesTest(FetcherTest): 2444 test_git_urls = { 2445 "git://tfs-example.org:22/tfs/example%20path/example.git;branch=master" : { 2446 'url': 'git://tfs-example.org:22/tfs/example%20path/example.git;branch=master', 2447 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example.git', 2448 'path': '/tfs/example path/example.git' 2449 }, 2450 "git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master" : { 2451 'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master', 2452 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example_repo.git', 2453 'path': '/tfs/example path/example repo.git' 2454 } 2455 } 2456 2457 def test_urls(self): 2458 2459 # Set fake SRCREV to stop git fetcher from trying to contact non-existent git repo 2460 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 2461 2462 for test_git_url, ref in self.test_git_urls.items(): 2463 2464 fetcher = bb.fetch.Fetch([test_git_url], self.d) 2465 ud = fetcher.ud[fetcher.urls[0]] 2466 2467 self.assertEqual(ud.url, ref['url']) 2468 self.assertEqual(ud.path, ref['path']) 2469 self.assertEqual(ud.localfile, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2470 self.assertEqual(ud.localpath, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2471 self.assertEqual(ud.lockfile, os.path.join(self.dldir, "git2", ref['gitsrcname'] + '.lock')) 2472 self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2473 self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz')) 2474 2475class CrateTest(FetcherTest): 2476 @skipIfNoNetwork() 2477 def test_crate_url(self): 2478 2479 uri = "crate://crates.io/glob/0.2.11" 2480 self.d.setVar('SRC_URI', uri) 2481 2482 uris = self.d.getVar('SRC_URI').split() 2483 d = self.d 2484 2485 fetcher = bb.fetch2.Fetch(uris, self.d) 2486 ud = fetcher.ud[fetcher.urls[0]] 2487 2488 self.assertIn("name", ud.parm) 2489 self.assertEqual(ud.parm["name"], "glob-0.2.11") 2490 self.assertIn("downloadfilename", ud.parm) 2491 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") 2492 2493 fetcher.download() 2494 fetcher.unpack(self.tempdir) 2495 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2496 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done']) 2497 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) 2498 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) 2499 2500 @skipIfNoNetwork() 2501 def test_crate_url_matching_recipe(self): 2502 2503 self.d.setVar('BP', 'glob-0.2.11') 2504 2505 uri = "crate://crates.io/glob/0.2.11" 2506 self.d.setVar('SRC_URI', uri) 2507 2508 uris = self.d.getVar('SRC_URI').split() 2509 d = self.d 2510 2511 fetcher = bb.fetch2.Fetch(uris, self.d) 2512 ud = fetcher.ud[fetcher.urls[0]] 2513 2514 self.assertIn("name", ud.parm) 2515 self.assertEqual(ud.parm["name"], "glob-0.2.11") 2516 self.assertIn("downloadfilename", ud.parm) 2517 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") 2518 2519 fetcher.download() 2520 fetcher.unpack(self.tempdir) 2521 self.assertEqual(sorted(os.listdir(self.tempdir)), ['download', 'glob-0.2.11', 'unpacked']) 2522 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done']) 2523 self.assertTrue(os.path.exists(self.tempdir + "/glob-0.2.11/src/lib.rs")) 2524 2525 @skipIfNoNetwork() 2526 def test_crate_url_params(self): 2527 2528 uri = "crate://crates.io/aho-corasick/0.7.20;name=aho-corasick-renamed" 2529 self.d.setVar('SRC_URI', uri) 2530 2531 uris = self.d.getVar('SRC_URI').split() 2532 d = self.d 2533 2534 fetcher = bb.fetch2.Fetch(uris, self.d) 2535 ud = fetcher.ud[fetcher.urls[0]] 2536 2537 self.assertIn("name", ud.parm) 2538 self.assertEqual(ud.parm["name"], "aho-corasick-renamed") 2539 self.assertIn("downloadfilename", ud.parm) 2540 self.assertEqual(ud.parm["downloadfilename"], "aho-corasick-0.7.20.crate") 2541 2542 fetcher.download() 2543 fetcher.unpack(self.tempdir) 2544 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2545 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['aho-corasick-0.7.20.crate', 'aho-corasick-0.7.20.crate.done']) 2546 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/.cargo-checksum.json")) 2547 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/src/lib.rs")) 2548 2549 @skipIfNoNetwork() 2550 def test_crate_url_multi(self): 2551 2552 uri = "crate://crates.io/glob/0.2.11 crate://crates.io/time/0.1.35" 2553 self.d.setVar('SRC_URI', uri) 2554 2555 uris = self.d.getVar('SRC_URI').split() 2556 d = self.d 2557 2558 fetcher = bb.fetch2.Fetch(uris, self.d) 2559 ud = fetcher.ud[fetcher.urls[0]] 2560 2561 self.assertIn("name", ud.parm) 2562 self.assertEqual(ud.parm["name"], "glob-0.2.11") 2563 self.assertIn("downloadfilename", ud.parm) 2564 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") 2565 2566 ud = fetcher.ud[fetcher.urls[1]] 2567 self.assertIn("name", ud.parm) 2568 self.assertEqual(ud.parm["name"], "time-0.1.35") 2569 self.assertIn("downloadfilename", ud.parm) 2570 self.assertEqual(ud.parm["downloadfilename"], "time-0.1.35.crate") 2571 2572 fetcher.download() 2573 fetcher.unpack(self.tempdir) 2574 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2575 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done', 'time-0.1.35.crate', 'time-0.1.35.crate.done']) 2576 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) 2577 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) 2578 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/.cargo-checksum.json")) 2579 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/src/lib.rs")) 2580 2581 @skipIfNoNetwork() 2582 def test_crate_incorrect_cksum(self): 2583 uri = "crate://crates.io/aho-corasick/0.7.20" 2584 self.d.setVar('SRC_URI', uri) 2585 self.d.setVarFlag("SRC_URI", "aho-corasick-0.7.20.sha256sum", hashlib.sha256("Invalid".encode("utf-8")).hexdigest()) 2586 2587 uris = self.d.getVar('SRC_URI').split() 2588 2589 fetcher = bb.fetch2.Fetch(uris, self.d) 2590 with self.assertRaisesRegex(bb.fetch2.FetchError, "Fetcher failure for URL"): 2591 fetcher.download() 2592 2593class NPMTest(FetcherTest): 2594 def skipIfNoNpm(): 2595 import shutil 2596 if not shutil.which('npm'): 2597 return unittest.skip('npm not installed') 2598 return lambda f: f 2599 2600 @skipIfNoNpm() 2601 @skipIfNoNetwork() 2602 def test_npm(self): 2603 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2604 fetcher = bb.fetch.Fetch([url], self.d) 2605 ud = fetcher.ud[fetcher.urls[0]] 2606 fetcher.download() 2607 self.assertTrue(os.path.exists(ud.localpath)) 2608 self.assertTrue(os.path.exists(ud.localpath + '.done')) 2609 self.assertTrue(os.path.exists(ud.resolvefile)) 2610 fetcher.unpack(self.unpackdir) 2611 unpackdir = os.path.join(self.unpackdir, 'npm') 2612 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2613 2614 @skipIfNoNpm() 2615 @skipIfNoNetwork() 2616 def test_npm_bad_checksum(self): 2617 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2618 # Fetch once to get a tarball 2619 fetcher = bb.fetch.Fetch([url], self.d) 2620 ud = fetcher.ud[fetcher.urls[0]] 2621 fetcher.download() 2622 self.assertTrue(os.path.exists(ud.localpath)) 2623 # Modify the tarball 2624 bad = b'bad checksum' 2625 with open(ud.localpath, 'wb') as f: 2626 f.write(bad) 2627 # Verify that the tarball is fetched again 2628 fetcher.download() 2629 badsum = hashlib.sha512(bad).hexdigest() 2630 self.assertTrue(os.path.exists(ud.localpath + '_bad-checksum_' + badsum)) 2631 self.assertTrue(os.path.exists(ud.localpath)) 2632 2633 @skipIfNoNpm() 2634 @skipIfNoNetwork() 2635 def test_npm_premirrors(self): 2636 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2637 # Fetch once to get a tarball 2638 fetcher = bb.fetch.Fetch([url], self.d) 2639 ud = fetcher.ud[fetcher.urls[0]] 2640 fetcher.download() 2641 self.assertTrue(os.path.exists(ud.localpath)) 2642 2643 # Setup the mirror by renaming the download directory 2644 mirrordir = os.path.join(self.tempdir, 'mirror') 2645 bb.utils.rename(self.dldir, mirrordir) 2646 os.mkdir(self.dldir) 2647 2648 # Configure the premirror to be used 2649 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/npm2' % mirrordir) 2650 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2651 2652 # Fetch again 2653 self.assertFalse(os.path.exists(ud.localpath)) 2654 # The npm fetcher doesn't handle that the .resolved file disappears 2655 # while the fetcher object exists, which it does when we rename the 2656 # download directory to "mirror" above. Thus we need a new fetcher to go 2657 # with the now empty download directory. 2658 fetcher = bb.fetch.Fetch([url], self.d) 2659 ud = fetcher.ud[fetcher.urls[0]] 2660 fetcher.download() 2661 self.assertTrue(os.path.exists(ud.localpath)) 2662 2663 @skipIfNoNpm() 2664 @skipIfNoNetwork() 2665 def test_npm_premirrors_with_specified_filename(self): 2666 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2667 # Fetch once to get a tarball 2668 fetcher = bb.fetch.Fetch([url], self.d) 2669 ud = fetcher.ud[fetcher.urls[0]] 2670 fetcher.download() 2671 self.assertTrue(os.path.exists(ud.localpath)) 2672 # Setup the mirror 2673 mirrordir = os.path.join(self.tempdir, 'mirror') 2674 bb.utils.mkdirhier(mirrordir) 2675 mirrorfilename = os.path.join(mirrordir, os.path.basename(ud.localpath)) 2676 os.replace(ud.localpath, mirrorfilename) 2677 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s' % mirrorfilename) 2678 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2679 # Fetch again 2680 self.assertFalse(os.path.exists(ud.localpath)) 2681 fetcher.download() 2682 self.assertTrue(os.path.exists(ud.localpath)) 2683 2684 @skipIfNoNpm() 2685 @skipIfNoNetwork() 2686 def test_npm_mirrors(self): 2687 # Fetch once to get a tarball 2688 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2689 fetcher = bb.fetch.Fetch([url], self.d) 2690 ud = fetcher.ud[fetcher.urls[0]] 2691 fetcher.download() 2692 self.assertTrue(os.path.exists(ud.localpath)) 2693 # Setup the mirror 2694 mirrordir = os.path.join(self.tempdir, 'mirror') 2695 bb.utils.mkdirhier(mirrordir) 2696 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 2697 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 2698 # Update the resolved url to an invalid url 2699 with open(ud.resolvefile, 'r') as f: 2700 url = f.read() 2701 uri = URI(url) 2702 uri.path = '/invalid' 2703 with open(ud.resolvefile, 'w') as f: 2704 f.write(str(uri)) 2705 # Fetch again 2706 self.assertFalse(os.path.exists(ud.localpath)) 2707 fetcher.download() 2708 self.assertTrue(os.path.exists(ud.localpath)) 2709 2710 @skipIfNoNpm() 2711 @skipIfNoNetwork() 2712 def test_npm_destsuffix_downloadfilename(self): 2713 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz' 2714 fetcher = bb.fetch.Fetch([url], self.d) 2715 fetcher.download() 2716 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'foo-bar.tgz'))) 2717 fetcher.unpack(self.unpackdir) 2718 unpackdir = os.path.join(self.unpackdir, 'foo', 'bar') 2719 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2720 2721 def test_npm_no_network_no_tarball(self): 2722 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2723 self.d.setVar('BB_NO_NETWORK', '1') 2724 fetcher = bb.fetch.Fetch([url], self.d) 2725 with self.assertRaises(bb.fetch2.NetworkAccess): 2726 fetcher.download() 2727 2728 @skipIfNoNpm() 2729 @skipIfNoNetwork() 2730 def test_npm_no_network_with_tarball(self): 2731 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2732 # Fetch once to get a tarball 2733 fetcher = bb.fetch.Fetch([url], self.d) 2734 fetcher.download() 2735 # Disable network access 2736 self.d.setVar('BB_NO_NETWORK', '1') 2737 # Fetch again 2738 fetcher.download() 2739 fetcher.unpack(self.unpackdir) 2740 unpackdir = os.path.join(self.unpackdir, 'npm') 2741 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2742 2743 @skipIfNoNpm() 2744 @skipIfNoNetwork() 2745 def test_npm_registry_alternate(self): 2746 url = 'npm://skimdb.npmjs.com;package=@savoirfairelinux/node-server-example;version=1.0.0' 2747 fetcher = bb.fetch.Fetch([url], self.d) 2748 fetcher.download() 2749 fetcher.unpack(self.unpackdir) 2750 unpackdir = os.path.join(self.unpackdir, 'npm') 2751 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2752 2753 @skipIfNoNpm() 2754 @skipIfNoNetwork() 2755 def test_npm_version_latest(self): 2756 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=latest' 2757 fetcher = bb.fetch.Fetch([url], self.d) 2758 fetcher.download() 2759 fetcher.unpack(self.unpackdir) 2760 unpackdir = os.path.join(self.unpackdir, 'npm') 2761 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2762 2763 @skipIfNoNpm() 2764 @skipIfNoNetwork() 2765 def test_npm_registry_invalid(self): 2766 url = 'npm://registry.invalid.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2767 fetcher = bb.fetch.Fetch([url], self.d) 2768 with self.assertRaises(bb.fetch2.FetchError): 2769 fetcher.download() 2770 2771 @skipIfNoNpm() 2772 @skipIfNoNetwork() 2773 def test_npm_package_invalid(self): 2774 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/invalid;version=1.0.0' 2775 fetcher = bb.fetch.Fetch([url], self.d) 2776 with self.assertRaises(bb.fetch2.FetchError): 2777 fetcher.download() 2778 2779 @skipIfNoNpm() 2780 @skipIfNoNetwork() 2781 def test_npm_version_invalid(self): 2782 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=invalid' 2783 with self.assertRaises(bb.fetch2.ParameterError): 2784 fetcher = bb.fetch.Fetch([url], self.d) 2785 2786 @skipIfNoNpm() 2787 @skipIfNoNetwork() 2788 def test_npm_registry_none(self): 2789 url = 'npm://;package=@savoirfairelinux/node-server-example;version=1.0.0' 2790 with self.assertRaises(bb.fetch2.MalformedUrl): 2791 fetcher = bb.fetch.Fetch([url], self.d) 2792 2793 @skipIfNoNpm() 2794 @skipIfNoNetwork() 2795 def test_npm_package_none(self): 2796 url = 'npm://registry.npmjs.org;version=1.0.0' 2797 with self.assertRaises(bb.fetch2.MissingParameterError): 2798 fetcher = bb.fetch.Fetch([url], self.d) 2799 2800 @skipIfNoNpm() 2801 @skipIfNoNetwork() 2802 def test_npm_version_none(self): 2803 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example' 2804 with self.assertRaises(bb.fetch2.MissingParameterError): 2805 fetcher = bb.fetch.Fetch([url], self.d) 2806 2807 def create_shrinkwrap_file(self, data): 2808 import json 2809 datadir = os.path.join(self.tempdir, 'data') 2810 swfile = os.path.join(datadir, 'npm-shrinkwrap.json') 2811 bb.utils.mkdirhier(datadir) 2812 with open(swfile, 'w') as f: 2813 json.dump(data, f) 2814 # Also configure the S directory 2815 self.sdir = os.path.join(self.unpackdir, 'S') 2816 self.d.setVar('S', self.sdir) 2817 return swfile 2818 2819 @skipIfNoNpm() 2820 @skipIfNoNetwork() 2821 def test_npmsw(self): 2822 swfile = self.create_shrinkwrap_file({ 2823 'dependencies': { 2824 'array-flatten': { 2825 'version': '1.1.1', 2826 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2827 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=', 2828 'dependencies': { 2829 'content-type': { 2830 'version': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', 2831 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', 2832 'dependencies': { 2833 'cookie': { 2834 'version': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09', 2835 'from': 'git+https://github.com/jshttp/cookie.git' 2836 } 2837 } 2838 } 2839 } 2840 } 2841 } 2842 }) 2843 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2844 fetcher.download() 2845 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2846 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2847 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) 2848 fetcher.unpack(self.unpackdir) 2849 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'npm-shrinkwrap.json'))) 2850 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json'))) 2851 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'package.json'))) 2852 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'node_modules', 'cookie', 'package.json'))) 2853 2854 @skipIfNoNpm() 2855 @skipIfNoNetwork() 2856 def test_npmsw_git(self): 2857 swfile = self.create_shrinkwrap_file({ 2858 'dependencies': { 2859 'cookie': { 2860 'version': 'github:jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09', 2861 'from': 'github:jshttp/cookie.git' 2862 } 2863 } 2864 }) 2865 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2866 fetcher.download() 2867 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) 2868 2869 swfile = self.create_shrinkwrap_file({ 2870 'dependencies': { 2871 'cookie': { 2872 'version': 'jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09', 2873 'from': 'jshttp/cookie.git' 2874 } 2875 } 2876 }) 2877 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2878 fetcher.download() 2879 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) 2880 2881 swfile = self.create_shrinkwrap_file({ 2882 'dependencies': { 2883 'nodejs': { 2884 'version': 'gitlab:gitlab-examples/nodejs.git#892a1f16725e56cc3a2cb0d677be42935c8fc262', 2885 'from': 'gitlab:gitlab-examples/nodejs' 2886 } 2887 } 2888 }) 2889 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2890 fetcher.download() 2891 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'gitlab.com.gitlab-examples.nodejs.git'))) 2892 2893 @skipIfNoNpm() 2894 @skipIfNoNetwork() 2895 def test_npmsw_dev(self): 2896 swfile = self.create_shrinkwrap_file({ 2897 'dependencies': { 2898 'array-flatten': { 2899 'version': '1.1.1', 2900 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2901 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2902 }, 2903 'content-type': { 2904 'version': '1.0.4', 2905 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', 2906 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', 2907 'dev': True 2908 } 2909 } 2910 }) 2911 # Fetch with dev disabled 2912 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2913 fetcher.download() 2914 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2915 self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2916 # Fetch with dev enabled 2917 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';dev=1'], self.d) 2918 fetcher.download() 2919 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2920 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2921 2922 @skipIfNoNpm() 2923 @skipIfNoNetwork() 2924 def test_npmsw_destsuffix(self): 2925 swfile = self.create_shrinkwrap_file({ 2926 'dependencies': { 2927 'array-flatten': { 2928 'version': '1.1.1', 2929 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2930 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2931 } 2932 } 2933 }) 2934 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';destsuffix=foo/bar'], self.d) 2935 fetcher.download() 2936 fetcher.unpack(self.unpackdir) 2937 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'foo', 'bar', 'node_modules', 'array-flatten', 'package.json'))) 2938 2939 def test_npmsw_no_network_no_tarball(self): 2940 swfile = self.create_shrinkwrap_file({ 2941 'dependencies': { 2942 'array-flatten': { 2943 'version': '1.1.1', 2944 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2945 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2946 } 2947 } 2948 }) 2949 self.d.setVar('BB_NO_NETWORK', '1') 2950 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2951 with self.assertRaises(bb.fetch2.NetworkAccess): 2952 fetcher.download() 2953 2954 @skipIfNoNpm() 2955 @skipIfNoNetwork() 2956 def test_npmsw_no_network_with_tarball(self): 2957 # Fetch once to get a tarball 2958 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 2959 fetcher.download() 2960 # Disable network access 2961 self.d.setVar('BB_NO_NETWORK', '1') 2962 # Fetch again 2963 swfile = self.create_shrinkwrap_file({ 2964 'dependencies': { 2965 'array-flatten': { 2966 'version': '1.1.1', 2967 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2968 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2969 } 2970 } 2971 }) 2972 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2973 fetcher.download() 2974 fetcher.unpack(self.unpackdir) 2975 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json'))) 2976 2977 @skipIfNoNpm() 2978 @skipIfNoNetwork() 2979 def test_npmsw_npm_reusability(self): 2980 # Fetch once with npmsw 2981 swfile = self.create_shrinkwrap_file({ 2982 'dependencies': { 2983 'array-flatten': { 2984 'version': '1.1.1', 2985 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2986 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2987 } 2988 } 2989 }) 2990 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2991 fetcher.download() 2992 # Disable network access 2993 self.d.setVar('BB_NO_NETWORK', '1') 2994 # Fetch again with npm 2995 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 2996 fetcher.download() 2997 fetcher.unpack(self.unpackdir) 2998 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm', 'package.json'))) 2999 3000 @skipIfNoNpm() 3001 @skipIfNoNetwork() 3002 def test_npmsw_bad_checksum(self): 3003 # Try to fetch with bad checksum 3004 swfile = self.create_shrinkwrap_file({ 3005 'dependencies': { 3006 'array-flatten': { 3007 'version': '1.1.1', 3008 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3009 'integrity': 'sha1-gfNEp2hqgLTFKT6P3AsBYMgsBqg=' 3010 } 3011 } 3012 }) 3013 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3014 with self.assertRaises(bb.fetch2.FetchError): 3015 fetcher.download() 3016 # Fetch correctly to get a tarball 3017 swfile = self.create_shrinkwrap_file({ 3018 'dependencies': { 3019 'array-flatten': { 3020 'version': '1.1.1', 3021 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3022 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3023 } 3024 } 3025 }) 3026 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3027 fetcher.download() 3028 localpath = os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz') 3029 self.assertTrue(os.path.exists(localpath)) 3030 # Modify the tarball 3031 bad = b'bad checksum' 3032 with open(localpath, 'wb') as f: 3033 f.write(bad) 3034 # Verify that the tarball is fetched again 3035 fetcher.download() 3036 badsum = hashlib.sha1(bad).hexdigest() 3037 self.assertTrue(os.path.exists(localpath + '_bad-checksum_' + badsum)) 3038 self.assertTrue(os.path.exists(localpath)) 3039 3040 @skipIfNoNpm() 3041 @skipIfNoNetwork() 3042 def test_npmsw_premirrors(self): 3043 # Fetch once to get a tarball 3044 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 3045 ud = fetcher.ud[fetcher.urls[0]] 3046 fetcher.download() 3047 self.assertTrue(os.path.exists(ud.localpath)) 3048 # Setup the mirror 3049 mirrordir = os.path.join(self.tempdir, 'mirror') 3050 bb.utils.mkdirhier(mirrordir) 3051 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 3052 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 3053 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 3054 # Fetch again 3055 self.assertFalse(os.path.exists(ud.localpath)) 3056 swfile = self.create_shrinkwrap_file({ 3057 'dependencies': { 3058 'array-flatten': { 3059 'version': '1.1.1', 3060 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3061 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3062 } 3063 } 3064 }) 3065 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3066 fetcher.download() 3067 self.assertTrue(os.path.exists(ud.localpath)) 3068 3069 @skipIfNoNpm() 3070 @skipIfNoNetwork() 3071 def test_npmsw_mirrors(self): 3072 # Fetch once to get a tarball 3073 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 3074 ud = fetcher.ud[fetcher.urls[0]] 3075 fetcher.download() 3076 self.assertTrue(os.path.exists(ud.localpath)) 3077 # Setup the mirror 3078 mirrordir = os.path.join(self.tempdir, 'mirror') 3079 bb.utils.mkdirhier(mirrordir) 3080 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 3081 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 3082 # Fetch again with invalid url 3083 self.assertFalse(os.path.exists(ud.localpath)) 3084 swfile = self.create_shrinkwrap_file({ 3085 'dependencies': { 3086 'array-flatten': { 3087 'version': '1.1.1', 3088 'resolved': 'https://invalid', 3089 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3090 } 3091 } 3092 }) 3093 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3094 fetcher.download() 3095 self.assertTrue(os.path.exists(ud.localpath)) 3096 3097class GitSharedTest(FetcherTest): 3098 def setUp(self): 3099 super(GitSharedTest, self).setUp() 3100 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 3101 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 3102 self.d.setVar("__BBSRCREV_SEEN", "1") 3103 3104 @skipIfNoNetwork() 3105 def test_shared_unpack(self): 3106 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3107 3108 fetcher.download() 3109 fetcher.unpack(self.unpackdir) 3110 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 3111 self.assertTrue(os.path.exists(alt)) 3112 3113 @skipIfNoNetwork() 3114 def test_noshared_unpack(self): 3115 self.d.setVar('BB_GIT_NOSHARED', '1') 3116 self.unpackdir += '_noshared' 3117 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3118 3119 fetcher.download() 3120 fetcher.unpack(self.unpackdir) 3121 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 3122 self.assertFalse(os.path.exists(alt)) 3123 3124 3125class FetchPremirroronlyLocalTest(FetcherTest): 3126 3127 def setUp(self): 3128 super(FetchPremirroronlyLocalTest, self).setUp() 3129 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3130 os.mkdir(self.mirrordir) 3131 self.reponame = "bitbake" 3132 self.gitdir = os.path.join(self.tempdir, "git", self.reponame) 3133 self.recipe_url = "git://git.fake.repo/bitbake;branch=master;protocol=https" 3134 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3135 self.d.setVar("BB_NO_NETWORK", "1") 3136 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3137 self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz" 3138 self.mirrorfile = os.path.join(self.mirrordir, self.mirrorname) 3139 self.testfilename = "bitbake-fetch.test" 3140 3141 def make_git_repo(self): 3142 recipeurl = "git:/git.fake.repo/bitbake" 3143 os.makedirs(self.gitdir) 3144 self.git_init(cwd=self.gitdir) 3145 for i in range(0): 3146 self.git_new_commit() 3147 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3148 3149 def git_new_commit(self): 3150 import random 3151 os.unlink(os.path.join(self.mirrordir, self.mirrorname)) 3152 branch = self.git("branch --show-current", self.gitdir).split() 3153 with open(os.path.join(self.gitdir, self.testfilename), "w") as testfile: 3154 testfile.write("File {} from branch {}; Useless random data {}".format(self.testfilename, branch, random.random())) 3155 self.git("add {}".format(self.testfilename), self.gitdir) 3156 self.git("commit -a -m \"This random commit {} in branch {}. I'm useless.\"".format(random.random(), branch), self.gitdir) 3157 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3158 return self.git("rev-parse HEAD", self.gitdir).strip() 3159 3160 def git_new_branch(self, name): 3161 self.git_new_commit() 3162 head = self.git("rev-parse HEAD", self.gitdir).strip() 3163 self.git("checkout -b {}".format(name), self.gitdir) 3164 newrev = self.git_new_commit() 3165 self.git("checkout {}".format(head), self.gitdir) 3166 return newrev 3167 3168 def test_mirror_multiple_fetches(self): 3169 self.make_git_repo() 3170 self.d.setVar("SRCREV", self.git_new_commit()) 3171 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3172 fetcher.download() 3173 fetcher.unpack(self.unpackdir) 3174 ## New commit in premirror. it's not in the download_dir 3175 self.d.setVar("SRCREV", self.git_new_commit()) 3176 fetcher2 = bb.fetch.Fetch([self.recipe_url], self.d) 3177 fetcher2.download() 3178 fetcher2.unpack(self.unpackdir) 3179 ## New commit in premirror. it's not in the download_dir 3180 self.d.setVar("SRCREV", self.git_new_commit()) 3181 fetcher3 = bb.fetch.Fetch([self.recipe_url], self.d) 3182 fetcher3.download() 3183 fetcher3.unpack(self.unpackdir) 3184 3185 3186 def test_mirror_commit_nonexistent(self): 3187 self.make_git_repo() 3188 self.d.setVar("SRCREV", "0"*40) 3189 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3190 with self.assertRaises(bb.fetch2.NetworkAccess): 3191 fetcher.download() 3192 3193 def test_mirror_commit_exists(self): 3194 self.make_git_repo() 3195 self.d.setVar("SRCREV", self.git_new_commit()) 3196 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3197 fetcher.download() 3198 fetcher.unpack(self.unpackdir) 3199 3200 def test_mirror_tarball_nonexistent(self): 3201 self.d.setVar("SRCREV", "0"*40) 3202 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3203 with self.assertRaises(bb.fetch2.NetworkAccess): 3204 fetcher.download() 3205 3206 def test_mirror_tarball_multiple_branches(self): 3207 """ 3208 test if PREMIRRORS can handle multiple name/branches correctly 3209 both branches have required revisions 3210 """ 3211 self.make_git_repo() 3212 branch1rev = self.git_new_branch("testbranch1") 3213 branch2rev = self.git_new_branch("testbranch2") 3214 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1,testbranch2;protocol=https;name=branch1,branch2" 3215 self.d.setVar("SRCREV_branch1", branch1rev) 3216 self.d.setVar("SRCREV_branch2", branch2rev) 3217 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3218 self.assertTrue(os.path.exists(self.mirrorfile), "Mirror file doesn't exist") 3219 fetcher.download() 3220 fetcher.unpack(os.path.join(self.tempdir, "unpacked")) 3221 unpacked = os.path.join(self.tempdir, "unpacked", "git", self.testfilename) 3222 self.assertTrue(os.path.exists(unpacked), "Repo has not been unpackaged properly!") 3223 with open(unpacked, 'r') as f: 3224 content = f.read() 3225 ## We expect to see testbranch1 in the file, not master, not testbranch2 3226 self.assertTrue(content.find("testbranch1") != -1, "Wrong branch has been checked out!") 3227 3228 def test_mirror_tarball_multiple_branches_nobranch(self): 3229 """ 3230 test if PREMIRRORS can handle multiple name/branches correctly 3231 Unbalanced name/branches raises ParameterError 3232 """ 3233 self.make_git_repo() 3234 branch1rev = self.git_new_branch("testbranch1") 3235 branch2rev = self.git_new_branch("testbranch2") 3236 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1;protocol=https;name=branch1,branch2" 3237 self.d.setVar("SRCREV_branch1", branch1rev) 3238 self.d.setVar("SRCREV_branch2", branch2rev) 3239 with self.assertRaises(bb.fetch2.ParameterError): 3240 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3241 3242 def test_mirror_tarball_multiple_branches_norev(self): 3243 """ 3244 test if PREMIRRORS can handle multiple name/branches correctly 3245 one of the branches specifies non existing SRCREV 3246 """ 3247 self.make_git_repo() 3248 branch1rev = self.git_new_branch("testbranch1") 3249 branch2rev = self.git_new_branch("testbranch2") 3250 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1,testbranch2;protocol=https;name=branch1,branch2" 3251 self.d.setVar("SRCREV_branch1", branch1rev) 3252 self.d.setVar("SRCREV_branch2", "0"*40) 3253 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3254 self.assertTrue(os.path.exists(self.mirrorfile), "Mirror file doesn't exist") 3255 with self.assertRaises(bb.fetch2.NetworkAccess): 3256 fetcher.download() 3257 3258 3259class FetchPremirroronlyNetworkTest(FetcherTest): 3260 3261 def setUp(self): 3262 super(FetchPremirroronlyNetworkTest, self).setUp() 3263 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3264 os.mkdir(self.mirrordir) 3265 self.reponame = "fstests" 3266 self.clonedir = os.path.join(self.tempdir, "git") 3267 self.gitdir = os.path.join(self.tempdir, "git", "{}.git".format(self.reponame)) 3268 self.recipe_url = "git://git.yoctoproject.org/fstests;protocol=https" 3269 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3270 self.d.setVar("BB_NO_NETWORK", "0") 3271 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3272 3273 def make_git_repo(self): 3274 import shutil 3275 self.mirrorname = "git2_git.yoctoproject.org.fstests.tar.gz" 3276 os.makedirs(self.clonedir) 3277 self.git("clone --bare --shallow-since=\"01.01.2013\" {}".format(self.recipe_url), self.clonedir) 3278 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3279 shutil.rmtree(self.clonedir) 3280 3281 @skipIfNoNetwork() 3282 def test_mirror_tarball_updated(self): 3283 self.make_git_repo() 3284 ## Upstream commit is in the mirror 3285 self.d.setVar("SRCREV", "49d65d53c2bf558ae6e9185af0f3af7b79d255ec") 3286 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3287 fetcher.download() 3288 3289 @skipIfNoNetwork() 3290 def test_mirror_tarball_outdated(self): 3291 self.make_git_repo() 3292 ## Upstream commit not in the mirror 3293 self.d.setVar("SRCREV", "15413486df1f5a5b5af699b6f3ba5f0984e52a9f") 3294 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3295 with self.assertRaises(bb.fetch2.NetworkAccess): 3296 fetcher.download() 3297 3298class FetchPremirroronlyMercurialTest(FetcherTest): 3299 """ Test for premirrors with mercurial repos 3300 the test covers also basic hg:// clone (see fetch_and_create_tarball 3301 """ 3302 def skipIfNoHg(): 3303 import shutil 3304 if not shutil.which('hg'): 3305 return unittest.skip('Mercurial not installed') 3306 return lambda f: f 3307 3308 def setUp(self): 3309 super(FetchPremirroronlyMercurialTest, self).setUp() 3310 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3311 os.mkdir(self.mirrordir) 3312 self.reponame = "libgnt" 3313 self.clonedir = os.path.join(self.tempdir, "hg") 3314 self.recipe_url = "hg://keep.imfreedom.org/libgnt;module=libgnt" 3315 self.d.setVar("SRCREV", "53e8b422faaf") 3316 self.mirrorname = "hg_libgnt_keep.imfreedom.org_.libgnt.tar.gz" 3317 3318 def fetch_and_create_tarball(self): 3319 """ 3320 Ask bitbake to download repo and prepare mirror tarball for us 3321 """ 3322 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") 3323 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3324 fetcher.download() 3325 mirrorfile = os.path.join(self.d.getVar("DL_DIR"), self.mirrorname) 3326 self.assertTrue(os.path.exists(mirrorfile), "Mirror tarball {} has not been created".format(mirrorfile)) 3327 ## moving tarball to mirror directory 3328 os.rename(mirrorfile, os.path.join(self.mirrordir, self.mirrorname)) 3329 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "0") 3330 3331 3332 @skipIfNoNetwork() 3333 @skipIfNoHg() 3334 def test_premirror_mercurial(self): 3335 self.fetch_and_create_tarball() 3336 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3337 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3338 self.d.setVar("BB_NO_NETWORK", "1") 3339 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3340 fetcher.download() 3341 3342class FetchPremirroronlyBrokenTarball(FetcherTest): 3343 3344 def setUp(self): 3345 super(FetchPremirroronlyBrokenTarball, self).setUp() 3346 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3347 os.mkdir(self.mirrordir) 3348 self.reponame = "bitbake" 3349 self.gitdir = os.path.join(self.tempdir, "git", self.reponame) 3350 self.recipe_url = "git://git.fake.repo/bitbake;protocol=https" 3351 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3352 self.d.setVar("BB_NO_NETWORK", "1") 3353 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3354 self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz" 3355 with open(os.path.join(self.mirrordir, self.mirrorname), 'w') as targz: 3356 targz.write("This is not tar.gz file!") 3357 3358 def test_mirror_broken_download(self): 3359 import sys 3360 self.d.setVar("SRCREV", "0"*40) 3361 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3362 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs() as logs: 3363 fetcher.download() 3364 output = "".join(logs.output) 3365 self.assertFalse(" not a git repository (or any parent up to mount point /)" in output) 3366