1# 2# BitBake Tests for the Fetcher (fetch2/) 3# 4# Copyright (C) 2012 Richard Purdie 5# 6# SPDX-License-Identifier: GPL-2.0-only 7# 8 9import contextlib 10import unittest 11import hashlib 12import tempfile 13import collections 14import os 15import signal 16import tarfile 17from bb.fetch2 import URI 18from bb.fetch2 import FetchMethod 19import bb 20from bb.tests.support.httpserver import HTTPService 21 22def skipIfNoNetwork(): 23 if os.environ.get("BB_SKIP_NETTESTS") == "yes": 24 return unittest.skip("network test") 25 return lambda f: f 26 27class TestTimeout(Exception): 28 # Indicate to pytest that this is not a test suite 29 __test__ = False 30 31class Timeout(): 32 33 def __init__(self, seconds): 34 self.seconds = seconds 35 36 def handle_timeout(self, signum, frame): 37 raise TestTimeout("Test failed: timeout reached") 38 39 def __enter__(self): 40 signal.signal(signal.SIGALRM, self.handle_timeout) 41 signal.alarm(self.seconds) 42 43 def __exit__(self, exc_type, exc_val, exc_tb): 44 signal.alarm(0) 45 46class URITest(unittest.TestCase): 47 test_uris = { 48 "http://www.google.com/index.html" : { 49 'uri': 'http://www.google.com/index.html', 50 'scheme': 'http', 51 'hostname': 'www.google.com', 52 'port': None, 53 'hostport': 'www.google.com', 54 'path': '/index.html', 55 'userinfo': '', 56 'username': '', 57 'password': '', 58 'params': {}, 59 'query': {}, 60 'relative': False 61 }, 62 "http://www.google.com/index.html;param1=value1" : { 63 'uri': 'http://www.google.com/index.html;param1=value1', 64 'scheme': 'http', 65 'hostname': 'www.google.com', 66 'port': None, 67 'hostport': 'www.google.com', 68 'path': '/index.html', 69 'userinfo': '', 70 'username': '', 71 'password': '', 72 'params': { 73 'param1': 'value1' 74 }, 75 'query': {}, 76 'relative': False 77 }, 78 "http://www.example.org/index.html?param1=value1" : { 79 'uri': 'http://www.example.org/index.html?param1=value1', 80 'scheme': 'http', 81 'hostname': 'www.example.org', 82 'port': None, 83 'hostport': 'www.example.org', 84 'path': '/index.html', 85 'userinfo': '', 86 'username': '', 87 'password': '', 88 'params': {}, 89 'query': { 90 'param1': 'value1' 91 }, 92 'relative': False 93 }, 94 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2" : { 95 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2', 96 'scheme': 'http', 97 'hostname': 'www.example.org', 98 'port': None, 99 'hostport': 'www.example.org', 100 'path': '/index.html', 101 'userinfo': '', 102 'username': '', 103 'password': '', 104 'params': { 105 'param2': 'value2' 106 }, 107 'query': { 108 'qparam1': 'qvalue1' 109 }, 110 'relative': False 111 }, 112 # Check that trailing semicolons are handled correctly 113 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2;" : { 114 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2', 115 'scheme': 'http', 116 'hostname': 'www.example.org', 117 'port': None, 118 'hostport': 'www.example.org', 119 'path': '/index.html', 120 'userinfo': '', 121 'username': '', 122 'password': '', 123 'params': { 124 'param2': 'value2' 125 }, 126 'query': { 127 'qparam1': 'qvalue1' 128 }, 129 'relative': False 130 }, 131 "http://www.example.com:8080/index.html" : { 132 'uri': 'http://www.example.com:8080/index.html', 133 'scheme': 'http', 134 'hostname': 'www.example.com', 135 'port': 8080, 136 'hostport': 'www.example.com:8080', 137 'path': '/index.html', 138 'userinfo': '', 139 'username': '', 140 'password': '', 141 'params': {}, 142 'query': {}, 143 'relative': False 144 }, 145 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : { 146 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg', 147 'scheme': 'cvs', 148 'hostname': 'cvs.handhelds.org', 149 'port': None, 150 'hostport': 'cvs.handhelds.org', 151 'path': '/cvs', 152 'userinfo': 'anoncvs', 153 'username': 'anoncvs', 154 'password': '', 155 'params': { 156 'module': 'familiar/dist/ipkg' 157 }, 158 'query': {}, 159 'relative': False 160 }, 161 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": { 162 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg', 163 'scheme': 'cvs', 164 'hostname': 'cvs.handhelds.org', 165 'port': None, 166 'hostport': 'cvs.handhelds.org', 167 'path': '/cvs', 168 'userinfo': 'anoncvs:anonymous', 169 'username': 'anoncvs', 170 'password': 'anonymous', 171 'params': collections.OrderedDict([ 172 ('tag', 'V0-99-81'), 173 ('module', 'familiar/dist/ipkg') 174 ]), 175 'query': {}, 176 'relative': False 177 }, 178 "file://example.diff": { # NOTE: Not RFC compliant! 179 'uri': 'file:example.diff', 180 'scheme': 'file', 181 'hostname': '', 182 'port': None, 183 'hostport': '', 184 'path': 'example.diff', 185 'userinfo': '', 186 'username': '', 187 'password': '', 188 'params': {}, 189 'query': {}, 190 'relative': True 191 }, 192 "file:example.diff": { # NOTE: RFC compliant version of the former 193 'uri': 'file:example.diff', 194 'scheme': 'file', 195 'hostname': '', 196 'port': None, 197 'hostport': '', 198 'path': 'example.diff', 199 'userinfo': '', 200 'userinfo': '', 201 'username': '', 202 'password': '', 203 'params': {}, 204 'query': {}, 205 'relative': True 206 }, 207 "file:///tmp/example.diff": { 208 'uri': 'file:///tmp/example.diff', 209 'scheme': 'file', 210 'hostname': '', 211 'port': None, 212 'hostport': '', 213 'path': '/tmp/example.diff', 214 'userinfo': '', 215 'userinfo': '', 216 'username': '', 217 'password': '', 218 'params': {}, 219 'query': {}, 220 'relative': False 221 }, 222 "git:///path/example.git": { 223 'uri': 'git:///path/example.git', 224 'scheme': 'git', 225 'hostname': '', 226 'port': None, 227 'hostport': '', 228 'path': '/path/example.git', 229 'userinfo': '', 230 'userinfo': '', 231 'username': '', 232 'password': '', 233 'params': {}, 234 'query': {}, 235 'relative': False 236 }, 237 "git:path/example.git": { 238 'uri': 'git:path/example.git', 239 'scheme': 'git', 240 'hostname': '', 241 'port': None, 242 'hostport': '', 243 'path': 'path/example.git', 244 'userinfo': '', 245 'userinfo': '', 246 'username': '', 247 'password': '', 248 'params': {}, 249 'query': {}, 250 'relative': True 251 }, 252 "git://example.net/path/example.git": { 253 'uri': 'git://example.net/path/example.git', 254 'scheme': 'git', 255 'hostname': 'example.net', 256 'port': None, 257 'hostport': 'example.net', 258 'path': '/path/example.git', 259 'userinfo': '', 260 'userinfo': '', 261 'username': '', 262 'password': '', 263 'params': {}, 264 'query': {}, 265 'relative': False 266 }, 267 "git://tfs-example.org:22/tfs/example%20path/example.git": { 268 'uri': 'git://tfs-example.org:22/tfs/example%20path/example.git', 269 'scheme': 'git', 270 'hostname': 'tfs-example.org', 271 'port': 22, 272 'hostport': 'tfs-example.org:22', 273 'path': '/tfs/example path/example.git', 274 'userinfo': '', 275 'userinfo': '', 276 'username': '', 277 'password': '', 278 'params': {}, 279 'query': {}, 280 'relative': False 281 }, 282 "http://somesite.net;someparam=1": { 283 'uri': 'http://somesite.net;someparam=1', 284 'scheme': 'http', 285 'hostname': 'somesite.net', 286 'port': None, 287 'hostport': 'somesite.net', 288 'path': '', 289 'userinfo': '', 290 'userinfo': '', 291 'username': '', 292 'password': '', 293 'params': {"someparam" : "1"}, 294 'query': {}, 295 'relative': False 296 }, 297 "file://somelocation;someparam=1": { 298 'uri': 'file:somelocation;someparam=1', 299 'scheme': 'file', 300 'hostname': '', 301 'port': None, 302 'hostport': '', 303 'path': 'somelocation', 304 'userinfo': '', 305 'userinfo': '', 306 'username': '', 307 'password': '', 308 'params': {"someparam" : "1"}, 309 'query': {}, 310 'relative': True 311 }, 312 "https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip": { 313 'uri': 'https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip', 314 'scheme': 'https', 315 'hostname': 'www.innodisk.com', 316 'port': None, 317 'hostport': 'www.innodisk.com', 318 'path': '/Download_file', 319 'userinfo': '', 320 'userinfo': '', 321 'username': '', 322 'password': '', 323 'params': {"downloadfilename" : "EGPL-T101.zip"}, 324 'query': {"9BE0BF6657": None}, 325 'relative': False 326 } 327 328 } 329 330 def test_uri(self): 331 for test_uri, ref in self.test_uris.items(): 332 uri = URI(test_uri) 333 334 self.assertEqual(str(uri), ref['uri']) 335 336 # expected attributes 337 self.assertEqual(uri.scheme, ref['scheme']) 338 339 self.assertEqual(uri.userinfo, ref['userinfo']) 340 self.assertEqual(uri.username, ref['username']) 341 self.assertEqual(uri.password, ref['password']) 342 343 self.assertEqual(uri.hostname, ref['hostname']) 344 self.assertEqual(uri.port, ref['port']) 345 self.assertEqual(uri.hostport, ref['hostport']) 346 347 self.assertEqual(uri.path, ref['path']) 348 self.assertEqual(uri.params, ref['params']) 349 350 self.assertEqual(uri.relative, ref['relative']) 351 352 def test_dict(self): 353 for test in self.test_uris.values(): 354 uri = URI() 355 356 self.assertEqual(uri.scheme, '') 357 self.assertEqual(uri.userinfo, '') 358 self.assertEqual(uri.username, '') 359 self.assertEqual(uri.password, '') 360 self.assertEqual(uri.hostname, '') 361 self.assertEqual(uri.port, None) 362 self.assertEqual(uri.path, '') 363 self.assertEqual(uri.params, {}) 364 365 366 uri.scheme = test['scheme'] 367 self.assertEqual(uri.scheme, test['scheme']) 368 369 uri.userinfo = test['userinfo'] 370 self.assertEqual(uri.userinfo, test['userinfo']) 371 self.assertEqual(uri.username, test['username']) 372 self.assertEqual(uri.password, test['password']) 373 374 # make sure changing the values doesn't do anything unexpected 375 uri.username = 'changeme' 376 self.assertEqual(uri.username, 'changeme') 377 self.assertEqual(uri.password, test['password']) 378 uri.password = 'insecure' 379 self.assertEqual(uri.username, 'changeme') 380 self.assertEqual(uri.password, 'insecure') 381 382 # reset back after our trickery 383 uri.userinfo = test['userinfo'] 384 self.assertEqual(uri.userinfo, test['userinfo']) 385 self.assertEqual(uri.username, test['username']) 386 self.assertEqual(uri.password, test['password']) 387 388 uri.hostname = test['hostname'] 389 self.assertEqual(uri.hostname, test['hostname']) 390 self.assertEqual(uri.hostport, test['hostname']) 391 392 uri.port = test['port'] 393 self.assertEqual(uri.port, test['port']) 394 self.assertEqual(uri.hostport, test['hostport']) 395 396 uri.path = test['path'] 397 self.assertEqual(uri.path, test['path']) 398 399 uri.params = test['params'] 400 self.assertEqual(uri.params, test['params']) 401 402 uri.query = test['query'] 403 self.assertEqual(uri.query, test['query']) 404 405 self.assertEqual(str(uri), test['uri']) 406 407 uri.params = {} 408 self.assertEqual(uri.params, {}) 409 self.assertEqual(str(uri), (str(uri).split(";"))[0]) 410 411class FetcherTest(unittest.TestCase): 412 413 def setUp(self): 414 self.origdir = os.getcwd() 415 self.d = bb.data.init() 416 self.tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-") 417 self.dldir = os.path.join(self.tempdir, "download") 418 os.mkdir(self.dldir) 419 self.d.setVar("DL_DIR", self.dldir) 420 self.unpackdir = os.path.join(self.tempdir, "unpacked") 421 os.mkdir(self.unpackdir) 422 persistdir = os.path.join(self.tempdir, "persistdata") 423 self.d.setVar("PERSISTENT_DIR", persistdir) 424 425 def tearDown(self): 426 os.chdir(self.origdir) 427 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes": 428 print("Not cleaning up %s. Please remove manually." % self.tempdir) 429 else: 430 bb.process.run('chmod u+rw -R %s' % self.tempdir) 431 bb.utils.prunedir(self.tempdir) 432 433 def git(self, cmd, cwd=None): 434 if isinstance(cmd, str): 435 cmd = 'git -c safe.bareRepository=all ' + cmd 436 else: 437 cmd = ['git', '-c', 'safe.bareRepository=all'] + cmd 438 if cwd is None: 439 cwd = self.gitdir 440 return bb.process.run(cmd, cwd=cwd)[0] 441 442 def git_init(self, cwd=None): 443 self.git('init', cwd=cwd) 444 # Explicitly set initial branch to master as 445 # a common setup is to use other default 446 # branch than master. 447 self.git(['checkout', '-b', 'master'], cwd=cwd) 448 449 try: 450 self.git(['config', 'user.email'], cwd=cwd) 451 except bb.process.ExecutionError: 452 self.git(['config', 'user.email', 'you@example.com'], cwd=cwd) 453 454 try: 455 self.git(['config', 'user.name'], cwd=cwd) 456 except bb.process.ExecutionError: 457 self.git(['config', 'user.name', 'Your Name'], cwd=cwd) 458 459class MirrorUriTest(FetcherTest): 460 461 replaceuris = { 462 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/") 463 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz", 464 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 465 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 466 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 467 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 468 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http") 469 : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 470 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake") 471 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890", 472 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache") 473 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 474 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/") 475 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 476 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3") 477 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", 478 ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz") 479 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", 480 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist") 481 : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", 482 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/") 483 : "file:///somepath/downloads/subversion-1.7.1.tar.bz2", 484 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 485 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 486 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 487 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 488 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http") 489 : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 490 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org") 491 : "http://somewhere2.org/somefile_1.2.3.tar.gz", 492 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/") 493 : "http://somewhere2.org/somefile_1.2.3.tar.gz", 494 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http") 495 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 496 ("git://user1@someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://user2@git.openembedded.org/bitbake;protocol=http") 497 : "git://user2@git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 498 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=git;branch=master", "git://someserver.org/bitbake", "git://someotherserver.org/bitbake;protocol=https") 499 : "git://someotherserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=https;branch=master", 500 ("gitsm://git.qemu.org/git/seabios.git/;protocol=https;name=roms/seabios;subpath=roms/seabios;bareclone=1;nobranch=1;rev=1234567890123456789012345678901234567890", "gitsm://.*/.*", "http://petalinux.xilinx.com/sswreleases/rel-v${XILINX_VER_MAIN}/downloads") : "http://petalinux.xilinx.com/sswreleases/rel-v%24%7BXILINX_VER_MAIN%7D/downloads/git2_git.qemu.org.git.seabios.git..tar.gz", 501 ("https://somewhere.org/example/1.0.0/example;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/PATH") 502 : "file:///mirror/example/1.0.0/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", 503 ("https://somewhere.org/example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/some-example-1.0.0.tgz") 504 : "file:///mirror/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", 505 506 #Renaming files doesn't work 507 #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz" 508 #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 509 } 510 511 mirrorvar = "http://.*/.* file:///somepath/downloads/ " \ 512 "git://someserver.org/bitbake git://git.openembedded.org/bitbake " \ 513 "https://.*/.* file:///someotherpath/downloads/ " \ 514 "http://.*/.* file:///someotherpath/downloads/ " \ 515 "svn://svn.server1.com/ svn://svn.server2.com/" 516 517 def test_urireplace(self): 518 self.d.setVar("FILESPATH", ".") 519 for k, v in self.replaceuris.items(): 520 ud = bb.fetch.FetchData(k[0], self.d) 521 ud.setup_localpath(self.d) 522 mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2])) 523 newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d) 524 self.assertEqual([v], newuris) 525 526 def test_urilist1(self): 527 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 528 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) 529 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 530 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz']) 531 532 def test_urilist2(self): 533 # Catch https:// -> files:// bug 534 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 535 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) 536 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 537 self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz']) 538 539 def test_urilistsvn(self): 540 # Catch svn:// -> svn:// bug 541 fetcher = bb.fetch.FetchData("svn://svn.server1.com/isource/svnroot/reponame/tags/tagname;module=path_in_tagnamefolder;protocol=https;rev=2", self.d) 542 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) 543 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 544 self.assertEqual(uris, ['svn://svn.server2.com/isource/svnroot/reponame/tags/tagname;module=path_in_tagnamefolder;protocol=https;rev=2']) 545 546 def test_mirror_of_mirror(self): 547 # Test if mirror of a mirror works 548 mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/" 549 mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/" 550 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 551 mirrors = bb.fetch2.mirror_from_string(mirrorvar) 552 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 553 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 554 'file:///someotherpath/downloads/bitbake-1.0.tar.gz', 555 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz', 556 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz']) 557 558 recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ " \ 559 "https://.*/[^/]* https://BBBB/B/B/B/" 560 561 def test_recursive(self): 562 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 563 mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar) 564 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 565 self.assertEqual(uris, ['http://AAAA/A/A/A/bitbake/bitbake-1.0.tar.gz', 566 'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz', 567 'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz']) 568 569 570class GitDownloadDirectoryNamingTest(FetcherTest): 571 def setUp(self): 572 super(GitDownloadDirectoryNamingTest, self).setUp() 573 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 574 self.recipe_dir = "git.openembedded.org.bitbake" 575 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 576 self.mirror_dir = "github.com.openembedded.bitbake.git" 577 578 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 579 580 def setup_mirror_rewrite(self): 581 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 582 583 @skipIfNoNetwork() 584 def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self): 585 self.setup_mirror_rewrite() 586 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 587 588 fetcher.download() 589 590 dir = os.listdir(self.dldir + "/git2") 591 self.assertIn(self.recipe_dir, dir) 592 593 @skipIfNoNetwork() 594 def test_that_directory_exists_for_mirrored_url_and_recipe_url_when_mirroring_is_used(self): 595 self.setup_mirror_rewrite() 596 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 597 598 fetcher.download() 599 600 dir = os.listdir(self.dldir + "/git2") 601 self.assertIn(self.mirror_dir, dir) 602 self.assertIn(self.recipe_dir, dir) 603 604 @skipIfNoNetwork() 605 def test_that_recipe_directory_and_mirrored_directory_exists_when_mirroring_is_used_and_the_mirrored_directory_already_exists(self): 606 self.setup_mirror_rewrite() 607 fetcher = bb.fetch.Fetch([self.mirror_url], self.d) 608 fetcher.download() 609 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 610 611 fetcher.download() 612 613 dir = os.listdir(self.dldir + "/git2") 614 self.assertIn(self.mirror_dir, dir) 615 self.assertIn(self.recipe_dir, dir) 616 617 618class TarballNamingTest(FetcherTest): 619 def setUp(self): 620 super(TarballNamingTest, self).setUp() 621 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 622 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz" 623 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 624 self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz" 625 626 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1') 627 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 628 629 def setup_mirror_rewrite(self): 630 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 631 632 @skipIfNoNetwork() 633 def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self): 634 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 635 636 fetcher.download() 637 638 dir = os.listdir(self.dldir) 639 self.assertIn(self.recipe_tarball, dir) 640 641 @skipIfNoNetwork() 642 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self): 643 self.setup_mirror_rewrite() 644 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 645 646 fetcher.download() 647 648 dir = os.listdir(self.dldir) 649 self.assertIn(self.mirror_tarball, dir) 650 651 652class GitShallowTarballNamingTest(FetcherTest): 653 def setUp(self): 654 super(GitShallowTarballNamingTest, self).setUp() 655 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 656 self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz" 657 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 658 self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz" 659 660 self.d.setVar('BB_GIT_SHALLOW', '1') 661 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 662 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 663 664 def setup_mirror_rewrite(self): 665 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 666 667 @skipIfNoNetwork() 668 def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self): 669 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 670 671 fetcher.download() 672 673 dir = os.listdir(self.dldir) 674 self.assertIn(self.recipe_tarball, dir) 675 676 @skipIfNoNetwork() 677 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self): 678 self.setup_mirror_rewrite() 679 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 680 681 fetcher.download() 682 683 dir = os.listdir(self.dldir) 684 self.assertIn(self.mirror_tarball, dir) 685 686 687class CleanTarballTest(FetcherTest): 688 def setUp(self): 689 super(CleanTarballTest, self).setUp() 690 self.recipe_url = "git://git.openembedded.org/bitbake;protocol=https" 691 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz" 692 693 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1') 694 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 695 696 @skipIfNoNetwork() 697 def test_that_the_tarball_contents_does_not_leak_info(self): 698 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 699 700 fetcher.download() 701 702 fetcher.unpack(self.unpackdir) 703 mtime = bb.process.run('git log --all -1 --format=%ct', 704 cwd=os.path.join(self.unpackdir, 'git')) 705 self.assertEqual(len(mtime), 2) 706 mtime = int(mtime[0]) 707 708 archive = tarfile.open(os.path.join(self.dldir, self.recipe_tarball)) 709 self.assertNotEqual(len(archive.members), 0) 710 for member in archive.members: 711 if member.name == ".": 712 continue 713 self.assertEqual(member.uname, 'oe', "user name for %s differs" % member.name) 714 self.assertEqual(member.uid, 0, "uid for %s differs" % member.name) 715 self.assertEqual(member.gname, 'oe', "group name for %s differs" % member.name) 716 self.assertEqual(member.gid, 0, "gid for %s differs" % member.name) 717 self.assertEqual(member.mtime, mtime, "mtime for %s differs" % member.name) 718 719 720class FetcherLocalTest(FetcherTest): 721 def setUp(self): 722 def touch(fn): 723 with open(fn, 'a'): 724 os.utime(fn, None) 725 726 super(FetcherLocalTest, self).setUp() 727 self.localsrcdir = os.path.join(self.tempdir, 'localsrc') 728 os.makedirs(self.localsrcdir) 729 touch(os.path.join(self.localsrcdir, 'a')) 730 touch(os.path.join(self.localsrcdir, 'b')) 731 os.makedirs(os.path.join(self.localsrcdir, 'dir')) 732 touch(os.path.join(self.localsrcdir, 'dir', 'c')) 733 touch(os.path.join(self.localsrcdir, 'dir', 'd')) 734 os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir')) 735 touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e')) 736 touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device')) 737 bb.process.run('tar cf archive.tar -C dir .', cwd=self.localsrcdir) 738 bb.process.run('tar czf archive.tar.gz -C dir .', cwd=self.localsrcdir) 739 bb.process.run('tar cjf archive.tar.bz2 -C dir .', cwd=self.localsrcdir) 740 self.d.setVar("FILESPATH", self.localsrcdir) 741 742 def fetchUnpack(self, uris): 743 fetcher = bb.fetch.Fetch(uris, self.d) 744 fetcher.download() 745 fetcher.unpack(self.unpackdir) 746 flst = [] 747 for root, dirs, files in os.walk(self.unpackdir): 748 for f in files: 749 flst.append(os.path.relpath(os.path.join(root, f), self.unpackdir)) 750 flst.sort() 751 return flst 752 753 def test_local_checksum_fails_no_file(self): 754 self.d.setVar("SRC_URI", "file://404") 755 with self.assertRaises(bb.BBHandledException): 756 bb.fetch.get_checksum_file_list(self.d) 757 758 def test_local(self): 759 tree = self.fetchUnpack(['file://a', 'file://dir/c']) 760 self.assertEqual(tree, ['a', 'dir/c']) 761 762 def test_local_backslash(self): 763 tree = self.fetchUnpack([r'file://backslash\x2dsystemd-unit.device']) 764 self.assertEqual(tree, [r'backslash\x2dsystemd-unit.device']) 765 766 def test_local_wildcard(self): 767 with self.assertRaises(bb.fetch2.ParameterError): 768 tree = self.fetchUnpack(['file://a', 'file://dir/*']) 769 770 def test_local_dir(self): 771 tree = self.fetchUnpack(['file://a', 'file://dir']) 772 self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e']) 773 774 def test_local_subdir(self): 775 tree = self.fetchUnpack(['file://dir/subdir']) 776 self.assertEqual(tree, ['dir/subdir/e']) 777 778 def test_local_subdir_file(self): 779 tree = self.fetchUnpack(['file://dir/subdir/e']) 780 self.assertEqual(tree, ['dir/subdir/e']) 781 782 def test_local_subdirparam(self): 783 tree = self.fetchUnpack(['file://a;subdir=bar', 'file://dir;subdir=foo/moo']) 784 self.assertEqual(tree, ['bar/a', 'foo/moo/dir/c', 'foo/moo/dir/d', 'foo/moo/dir/subdir/e']) 785 786 def test_local_deepsubdirparam(self): 787 tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar']) 788 self.assertEqual(tree, ['bar/dir/subdir/e']) 789 790 def test_local_absolutedir(self): 791 # Unpacking to an absolute path that is a subdirectory of the root 792 # should work 793 tree = self.fetchUnpack(['file://a;subdir=%s' % os.path.join(self.unpackdir, 'bar')]) 794 795 # Unpacking to an absolute path outside of the root should fail 796 with self.assertRaises(bb.fetch2.UnpackError): 797 self.fetchUnpack(['file://a;subdir=/bin/sh']) 798 799 def test_local_striplevel(self): 800 tree = self.fetchUnpack(['file://archive.tar;subdir=bar;striplevel=1']) 801 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 802 803 def test_local_striplevel_gzip(self): 804 tree = self.fetchUnpack(['file://archive.tar.gz;subdir=bar;striplevel=1']) 805 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 806 807 def test_local_striplevel_bzip2(self): 808 tree = self.fetchUnpack(['file://archive.tar.bz2;subdir=bar;striplevel=1']) 809 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 810 811 def dummyGitTest(self, suffix): 812 # Create dummy local Git repo 813 src_dir = tempfile.mkdtemp(dir=self.tempdir, 814 prefix='gitfetch_localusehead_') 815 self.gitdir = os.path.abspath(src_dir) 816 self.git_init() 817 self.git(['commit', '--allow-empty', '-m', 'Dummy commit']) 818 # Use other branch than master 819 self.git(['checkout', '-b', 'my-devel']) 820 self.git(['commit', '--allow-empty', '-m', 'Dummy commit 2']) 821 orig_rev = self.git(['rev-parse', 'HEAD']).strip() 822 823 # Fetch and check revision 824 self.d.setVar("SRCREV", "AUTOINC") 825 self.d.setVar("__BBSRCREV_SEEN", "1") 826 url = "git://" + self.gitdir + ";branch=master;protocol=file;" + suffix 827 fetcher = bb.fetch.Fetch([url], self.d) 828 fetcher.download() 829 fetcher.unpack(self.unpackdir) 830 unpack_rev = self.git(['rev-parse', 'HEAD'], 831 cwd=os.path.join(self.unpackdir, 'git')).strip() 832 self.assertEqual(orig_rev, unpack_rev) 833 834 def test_local_gitfetch_usehead(self): 835 self.dummyGitTest("usehead=1") 836 837 def test_local_gitfetch_usehead_withname(self): 838 self.dummyGitTest("usehead=1;name=newName") 839 840 def test_local_gitfetch_shared(self): 841 self.dummyGitTest("usehead=1;name=sharedName") 842 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 843 self.assertTrue(os.path.exists(alt)) 844 845 def test_local_gitfetch_noshared(self): 846 self.d.setVar('BB_GIT_NOSHARED', '1') 847 self.unpackdir += '_noshared' 848 self.dummyGitTest("usehead=1;name=noSharedName") 849 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 850 self.assertFalse(os.path.exists(alt)) 851 852class FetcherNoNetworkTest(FetcherTest): 853 def setUp(self): 854 super().setUp() 855 # all test cases are based on not having network 856 self.d.setVar("BB_NO_NETWORK", "1") 857 858 def test_missing(self): 859 string = "this is a test file\n".encode("utf-8") 860 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 861 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 862 863 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 864 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 865 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 866 with self.assertRaises(bb.fetch2.NetworkAccess): 867 fetcher.download() 868 869 def test_valid_missing_donestamp(self): 870 # create the file in the download directory with correct hash 871 string = "this is a test file\n".encode("utf-8") 872 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb") as f: 873 f.write(string) 874 875 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 876 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 877 878 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 879 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 880 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 881 fetcher.download() 882 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 883 884 def test_invalid_missing_donestamp(self): 885 # create an invalid file in the download directory with incorrect hash 886 string = "this is a test file\n".encode("utf-8") 887 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 888 pass 889 890 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 891 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 892 893 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 894 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 895 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 896 with self.assertRaises(bb.fetch2.NetworkAccess): 897 fetcher.download() 898 # the existing file should not exist or should have be moved to "bad-checksum" 899 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 900 901 def test_nochecksums_missing(self): 902 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 903 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 904 # ssh fetch does not support checksums 905 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 906 # attempts to download with missing donestamp 907 with self.assertRaises(bb.fetch2.NetworkAccess): 908 fetcher.download() 909 910 def test_nochecksums_missing_donestamp(self): 911 # create a file in the download directory 912 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 913 pass 914 915 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 916 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 917 # ssh fetch does not support checksums 918 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 919 # attempts to download with missing donestamp 920 with self.assertRaises(bb.fetch2.NetworkAccess): 921 fetcher.download() 922 923 def test_nochecksums_has_donestamp(self): 924 # create a file in the download directory with the donestamp 925 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 926 pass 927 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"): 928 pass 929 930 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 931 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 932 # ssh fetch does not support checksums 933 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 934 # should not fetch 935 fetcher.download() 936 # both files should still exist 937 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 938 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 939 940 def test_nochecksums_missing_has_donestamp(self): 941 # create a file in the download directory with the donestamp 942 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"): 943 pass 944 945 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 946 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 947 # ssh fetch does not support checksums 948 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 949 with self.assertRaises(bb.fetch2.NetworkAccess): 950 fetcher.download() 951 # both files should still exist 952 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 953 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 954 955class FetcherNetworkTest(FetcherTest): 956 @skipIfNoNetwork() 957 def test_fetch(self): 958 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) 959 fetcher.download() 960 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 961 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892) 962 self.d.setVar("BB_NO_NETWORK", "1") 963 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) 964 fetcher.download() 965 fetcher.unpack(self.unpackdir) 966 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9) 967 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9) 968 969 @skipIfNoNetwork() 970 def test_fetch_mirror(self): 971 self.d.setVar("MIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 972 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 973 fetcher.download() 974 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 975 976 @skipIfNoNetwork() 977 def test_fetch_mirror_of_mirror(self): 978 self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ http://invalid2.yoctoproject.org/.* https://downloads.yoctoproject.org/releases/bitbake") 979 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 980 fetcher.download() 981 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 982 983 @skipIfNoNetwork() 984 def test_fetch_file_mirror_of_mirror(self): 985 self.d.setVar("FILESPATH", ".") 986 self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ file:///some1where/.* file://some2where/ file://some2where/.* https://downloads.yoctoproject.org/releases/bitbake") 987 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 988 os.mkdir(self.dldir + "/some2where") 989 fetcher.download() 990 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 991 992 @skipIfNoNetwork() 993 def test_fetch_premirror(self): 994 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 995 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 996 fetcher.download() 997 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 998 999 @skipIfNoNetwork() 1000 def test_fetch_specify_downloadfilename(self): 1001 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d) 1002 fetcher.download() 1003 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749) 1004 1005 @skipIfNoNetwork() 1006 def test_fetch_premirror_specify_downloadfilename_regex_uri(self): 1007 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake/") 1008 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 1009 fetcher.download() 1010 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1011 1012 @skipIfNoNetwork() 1013 # BZ13039 1014 def test_fetch_premirror_specify_downloadfilename_specific_uri(self): 1015 self.d.setVar("PREMIRRORS", "http://invalid.yoctoproject.org/releases/bitbake https://downloads.yoctoproject.org/releases/bitbake") 1016 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 1017 fetcher.download() 1018 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1019 1020 @skipIfNoNetwork() 1021 def test_fetch_premirror_use_downloadfilename_to_fetch(self): 1022 # Ensure downloadfilename is used when fetching from premirror. 1023 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 1024 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 1025 fetcher.download() 1026 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 1027 1028 @skipIfNoNetwork() 1029 def gitfetcher(self, url1, url2): 1030 def checkrevision(self, fetcher): 1031 fetcher.unpack(self.unpackdir) 1032 revision = self.git(['rev-parse', 'HEAD'], 1033 cwd=os.path.join(self.unpackdir, 'git')).strip() 1034 self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5") 1035 1036 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") 1037 self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5") 1038 fetcher = bb.fetch.Fetch([url1], self.d) 1039 fetcher.download() 1040 checkrevision(self, fetcher) 1041 # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works 1042 bb.utils.prunedir(self.dldir + "/git2/") 1043 bb.utils.prunedir(self.unpackdir) 1044 self.d.setVar("BB_NO_NETWORK", "1") 1045 fetcher = bb.fetch.Fetch([url2], self.d) 1046 fetcher.download() 1047 checkrevision(self, fetcher) 1048 1049 @skipIfNoNetwork() 1050 def test_gitfetch(self): 1051 url1 = url2 = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 1052 self.gitfetcher(url1, url2) 1053 1054 @skipIfNoNetwork() 1055 def test_gitfetch_goodsrcrev(self): 1056 # SRCREV is set but matches rev= parameter 1057 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https" 1058 self.gitfetcher(url1, url2) 1059 1060 @skipIfNoNetwork() 1061 def test_gitfetch_badsrcrev(self): 1062 # SRCREV is set but does not match rev= parameter 1063 url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https" 1064 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) 1065 1066 @skipIfNoNetwork() 1067 def test_gitfetch_tagandrev(self): 1068 # SRCREV is set but does not match rev= parameter 1069 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5;protocol=https" 1070 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) 1071 1072 @skipIfNoNetwork() 1073 def test_gitfetch_usehead(self): 1074 # Since self.gitfetcher() sets SRCREV we expect this to override 1075 # `usehead=1' and instead fetch the specified SRCREV. See 1076 # test_local_gitfetch_usehead() for a positive use of the usehead 1077 # feature. 1078 url = "git://git.openembedded.org/bitbake;usehead=1;branch=master;protocol=https" 1079 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url) 1080 1081 @skipIfNoNetwork() 1082 def test_gitfetch_usehead_withname(self): 1083 # Since self.gitfetcher() sets SRCREV we expect this to override 1084 # `usehead=1' and instead fetch the specified SRCREV. See 1085 # test_local_gitfetch_usehead() for a positive use of the usehead 1086 # feature. 1087 url = "git://git.openembedded.org/bitbake;usehead=1;name=newName;branch=master;protocol=https" 1088 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url) 1089 1090 @skipIfNoNetwork() 1091 def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self): 1092 recipeurl = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 1093 mirrorurl = "git://someserver.org/bitbake;branch=master;protocol=https" 1094 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake") 1095 self.gitfetcher(recipeurl, mirrorurl) 1096 1097 @skipIfNoNetwork() 1098 def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self): 1099 recipeurl = "git://someserver.org/bitbake;branch=master;protocol=https" 1100 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake") 1101 self.gitfetcher(recipeurl, recipeurl) 1102 1103 @skipIfNoNetwork() 1104 def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self): 1105 realurl = "https://git.openembedded.org/bitbake" 1106 recipeurl = "git://someserver.org/bitbake;protocol=https" 1107 self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git") 1108 os.chdir(self.tempdir) 1109 self.git(['clone', realurl, self.sourcedir], cwd=self.tempdir) 1110 self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file" % (recipeurl, self.sourcedir)) 1111 self.gitfetcher(recipeurl, recipeurl) 1112 1113 @skipIfNoNetwork() 1114 def test_git_submodule(self): 1115 # URL with ssh submodules 1116 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7;branch=master;protocol=https" 1117 # Original URL (comment this if you have ssh access to git.yoctoproject.org) 1118 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee;branch=master;protocol=https" 1119 fetcher = bb.fetch.Fetch([url], self.d) 1120 fetcher.download() 1121 # Previous cwd has been deleted 1122 os.chdir(os.path.dirname(self.unpackdir)) 1123 fetcher.unpack(self.unpackdir) 1124 1125 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1126 self.assertTrue(os.path.exists(repo_path), msg='Unpacked repository missing') 1127 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake')), msg='bitbake submodule missing') 1128 self.assertFalse(os.path.exists(os.path.join(repo_path, 'na')), msg='uninitialized submodule present') 1129 1130 # Only when we're running the extended test with a submodule's submodule, can we check this. 1131 if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')): 1132 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing') 1133 1134 @skipIfNoNetwork() 1135 def test_git_submodule_restricted_network_premirrors(self): 1136 # this test is to ensure that premirrors will be tried in restricted network 1137 # that is, BB_ALLOWED_NETWORKS does not contain the domain the url uses 1138 url = "gitsm://github.com/grpc/grpc.git;protocol=https;name=grpc;branch=v1.60.x;rev=0ef13a7555dbaadd4633399242524129eef5e231" 1139 # create a download directory to be used as premirror later 1140 tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-") 1141 dl_premirror = os.path.join(tempdir, "download-premirror") 1142 os.mkdir(dl_premirror) 1143 self.d.setVar("DL_DIR", dl_premirror) 1144 fetcher = bb.fetch.Fetch([url], self.d) 1145 fetcher.download() 1146 # now use the premirror in restricted network 1147 self.d.setVar("DL_DIR", self.dldir) 1148 self.d.setVar("PREMIRRORS", "gitsm://.*/.* gitsm://%s/git2/MIRRORNAME;protocol=file" % dl_premirror) 1149 self.d.setVar("BB_ALLOWED_NETWORKS", "*.some.domain") 1150 fetcher = bb.fetch.Fetch([url], self.d) 1151 fetcher.download() 1152 1153 @skipIfNoNetwork() 1154 def test_git_submodule_dbus_broker(self): 1155 # The following external repositories have show failures in fetch and unpack operations 1156 # We want to avoid regressions! 1157 url = "gitsm://github.com/bus1/dbus-broker;protocol=https;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main" 1158 fetcher = bb.fetch.Fetch([url], self.d) 1159 fetcher.download() 1160 # Previous cwd has been deleted 1161 os.chdir(os.path.dirname(self.unpackdir)) 1162 fetcher.unpack(self.unpackdir) 1163 1164 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1165 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-dvar/config')), msg='Missing submodule config "subprojects/c-dvar"') 1166 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-list/config')), msg='Missing submodule config "subprojects/c-list"') 1167 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-rbtree/config')), msg='Missing submodule config "subprojects/c-rbtree"') 1168 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"') 1169 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"') 1170 1171 @skipIfNoNetwork() 1172 def test_git_submodule_CLI11(self): 1173 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main" 1174 fetcher = bb.fetch.Fetch([url], self.d) 1175 fetcher.download() 1176 # Previous cwd has been deleted 1177 os.chdir(os.path.dirname(self.unpackdir)) 1178 fetcher.unpack(self.unpackdir) 1179 1180 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1181 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"') 1182 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') 1183 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') 1184 1185 @skipIfNoNetwork() 1186 def test_git_submodule_update_CLI11(self): 1187 """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """ 1188 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main" 1189 fetcher = bb.fetch.Fetch([url], self.d) 1190 fetcher.download() 1191 1192 # CLI11 that pulls in a newer nlohmann-json 1193 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main" 1194 fetcher = bb.fetch.Fetch([url], self.d) 1195 fetcher.download() 1196 # Previous cwd has been deleted 1197 os.chdir(os.path.dirname(self.unpackdir)) 1198 fetcher.unpack(self.unpackdir) 1199 1200 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1201 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"') 1202 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') 1203 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') 1204 1205 @skipIfNoNetwork() 1206 def test_git_submodule_aktualizr(self): 1207 url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=https;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44" 1208 fetcher = bb.fetch.Fetch([url], self.d) 1209 fetcher.download() 1210 # Previous cwd has been deleted 1211 os.chdir(os.path.dirname(self.unpackdir)) 1212 fetcher.unpack(self.unpackdir) 1213 1214 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1215 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule config "partial/extern/isotp-c/config"') 1216 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')), msg='Missing submodule config "partial/extern/isotp-c/modules/deps/bitfield-c/config"') 1217 self.assertTrue(os.path.exists(os.path.join(repo_path, 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of submodule isotp-c did not unpack properly") 1218 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule config "tests/tuf-test-vectors/config"') 1219 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"') 1220 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"') 1221 1222 @skipIfNoNetwork() 1223 def test_git_submodule_iotedge(self): 1224 """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """ 1225 1226 # This repository also has submodules where the module (name), path and url do not align 1227 url = "gitsm://github.com/azure/iotedge.git;protocol=https;rev=d76e0316c6f324345d77c48a83ce836d09392699;branch=main" 1228 fetcher = bb.fetch.Fetch([url], self.d) 1229 fetcher.download() 1230 # Previous cwd has been deleted 1231 os.chdir(os.path.dirname(self.unpackdir)) 1232 fetcher.unpack(self.unpackdir) 1233 1234 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1235 1236 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/README.md')), msg='Missing submodule checkout') 1237 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/ctest/README.md')), msg='Missing submodule checkout') 1238 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/testrunner/readme.md')), msg='Missing submodule checkout') 1239 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/readme.md')), msg='Missing submodule checkout') 1240 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') 1241 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') 1242 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/README.md')), msg='Missing submodule checkout') 1243 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/README.md')), msg='Missing submodule checkout') 1244 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/ctest/README.md')), msg='Missing submodule checkout') 1245 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/testrunner/readme.md')), msg='Missing submodule checkout') 1246 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/readme.md')), msg='Missing submodule checkout') 1247 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') 1248 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') 1249 1250 @skipIfNoNetwork() 1251 def test_git_submodule_reference_to_parent(self): 1252 self.recipe_url = "gitsm://github.com/gflags/gflags.git;protocol=https;branch=master" 1253 self.d.setVar("SRCREV", "14e1138441bbbb584160cb1c0a0426ec1bac35f1") 1254 with Timeout(60): 1255 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 1256 with self.assertRaises(bb.fetch2.FetchError): 1257 fetcher.download() 1258 1259class SVNTest(FetcherTest): 1260 def skipIfNoSvn(): 1261 import shutil 1262 if not shutil.which("svn"): 1263 return unittest.skip("svn not installed, tests being skipped") 1264 1265 if not shutil.which("svnadmin"): 1266 return unittest.skip("svnadmin not installed, tests being skipped") 1267 1268 return lambda f: f 1269 1270 @skipIfNoSvn() 1271 def setUp(self): 1272 """ Create a local repository """ 1273 1274 super(SVNTest, self).setUp() 1275 1276 # Create something we can fetch 1277 src_dir = tempfile.mkdtemp(dir=self.tempdir, 1278 prefix='svnfetch_srcdir_') 1279 src_dir = os.path.abspath(src_dir) 1280 bb.process.run("echo readme > README.md", cwd=src_dir) 1281 1282 # Store it in a local SVN repository 1283 repo_dir = tempfile.mkdtemp(dir=self.tempdir, 1284 prefix='svnfetch_localrepo_') 1285 repo_dir = os.path.abspath(repo_dir) 1286 bb.process.run("svnadmin create project", cwd=repo_dir) 1287 1288 self.repo_url = "file://%s/project" % repo_dir 1289 bb.process.run("svn import --non-interactive -m 'Initial import' %s %s/trunk" % (src_dir, self.repo_url), 1290 cwd=repo_dir) 1291 1292 bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir) 1293 # Github won't emulate SVN anymore (see https://github.blog/2023-01-20-sunsetting-subversion-support/) 1294 # Use still accessible svn repo (only trunk to avoid longer downloads) 1295 bb.process.run("svn propset svn:externals 'bitbake https://svn.apache.org/repos/asf/serf/trunk' .", 1296 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) 1297 bb.process.run("svn commit --non-interactive -m 'Add external'", 1298 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) 1299 1300 self.src_dir = src_dir 1301 self.repo_dir = repo_dir 1302 1303 @skipIfNoSvn() 1304 def tearDown(self): 1305 os.chdir(self.origdir) 1306 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes": 1307 print("Not cleaning up %s. Please remove manually." % self.tempdir) 1308 else: 1309 bb.utils.prunedir(self.tempdir) 1310 1311 @skipIfNoSvn() 1312 @skipIfNoNetwork() 1313 def test_noexternal_svn(self): 1314 # Always match the rev count from setUp (currently rev 2) 1315 url = "svn://%s;module=trunk;protocol=file;rev=2" % self.repo_url.replace('file://', '') 1316 fetcher = bb.fetch.Fetch([url], self.d) 1317 fetcher.download() 1318 os.chdir(os.path.dirname(self.unpackdir)) 1319 fetcher.unpack(self.unpackdir) 1320 1321 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") 1322 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") 1323 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should NOT exist") 1324 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should NOT exit") 1325 1326 @skipIfNoSvn() 1327 def test_external_svn(self): 1328 # Always match the rev count from setUp (currently rev 2) 1329 url = "svn://%s;module=trunk;protocol=file;externals=allowed;rev=2" % self.repo_url.replace('file://', '') 1330 fetcher = bb.fetch.Fetch([url], self.d) 1331 fetcher.download() 1332 os.chdir(os.path.dirname(self.unpackdir)) 1333 fetcher.unpack(self.unpackdir) 1334 1335 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") 1336 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") 1337 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should exist") 1338 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should exit") 1339 1340class TrustedNetworksTest(FetcherTest): 1341 def test_trusted_network(self): 1342 # Ensure trusted_network returns False when the host IS in the list. 1343 url = "git://Someserver.org/foo;rev=1;branch=master" 1344 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org") 1345 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1346 1347 def test_wild_trusted_network(self): 1348 # Ensure trusted_network returns true when the *.host IS in the list. 1349 url = "git://Someserver.org/foo;rev=1;branch=master" 1350 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1351 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1352 1353 def test_prefix_wild_trusted_network(self): 1354 # Ensure trusted_network returns true when the prefix matches *.host. 1355 url = "git://git.Someserver.org/foo;rev=1;branch=master" 1356 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1357 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1358 1359 def test_two_prefix_wild_trusted_network(self): 1360 # Ensure trusted_network returns true when the prefix matches *.host. 1361 url = "git://something.git.Someserver.org/foo;rev=1;branch=master" 1362 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1363 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1364 1365 def test_port_trusted_network(self): 1366 # Ensure trusted_network returns True, even if the url specifies a port. 1367 url = "git://someserver.org:8080/foo;rev=1;branch=master" 1368 self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org") 1369 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1370 1371 def test_untrusted_network(self): 1372 # Ensure trusted_network returns False when the host is NOT in the list. 1373 url = "git://someserver.org/foo;rev=1;branch=master" 1374 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org") 1375 self.assertFalse(bb.fetch.trusted_network(self.d, url)) 1376 1377 def test_wild_untrusted_network(self): 1378 # Ensure trusted_network returns False when the host is NOT in the list. 1379 url = "git://*.someserver.org/foo;rev=1;branch=master" 1380 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org") 1381 self.assertFalse(bb.fetch.trusted_network(self.d, url)) 1382 1383class URLHandle(unittest.TestCase): 1384 1385 datatable = { 1386 "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}), 1387 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}), 1388 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])), 1389 "git://git.openembedded.org/bitbake;branch=@foo;protocol=https" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo', 'protocol' : 'https'}), 1390 "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}), 1391 "https://somesite.com/somerepo.git;user=anyUser:idtoken=1234" : ('https', 'somesite.com', '/somerepo.git', '', '', {'user': 'anyUser:idtoken=1234'}), 1392 r'git://s.o-me_ONE:!#$%^&*()-_={}[]\|:?,.<>~`@git.openembedded.org/bitbake;branch=main;protocol=https': ('git', 'git.openembedded.org', '/bitbake', 's.o-me_ONE', r'!#$%^&*()-_={}[]\|:?,.<>~`', {'branch': 'main', 'protocol' : 'https'}), 1393 } 1394 # we require a pathname to encodeurl but users can still pass such urls to 1395 # decodeurl and we need to handle them 1396 decodedata = datatable.copy() 1397 decodedata.update({ 1398 "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}), 1399 "npmsw://some.registry.url;package=@pkg;version=latest": ('npmsw', 'some.registry.url', '/', '', '', {'package': '@pkg', 'version': 'latest'}), 1400 }) 1401 1402 def test_decodeurl(self): 1403 for k, v in self.decodedata.items(): 1404 result = bb.fetch.decodeurl(k) 1405 self.assertEqual(result, v) 1406 1407 def test_encodeurl(self): 1408 for k, v in self.datatable.items(): 1409 result = bb.fetch.encodeurl(v) 1410 self.assertEqual(result, k) 1411 1412class FetchLatestVersionTest(FetcherTest): 1413 1414 test_git_uris = { 1415 # version pattern "X.Y.Z" 1416 ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "", "") 1417 : "1.99.4", 1418 # version pattern "vX.Y" 1419 # mirror of git.infradead.org since network issues interfered with testing 1420 ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git;branch=master;protocol=https", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "", "") 1421 : "1.5.0", 1422 # version pattern "pkg_name-X.Y" 1423 # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing 1424 ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto;branch=master;protocol=https", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "", "") 1425 : "1.0", 1426 # version pattern "pkg_name-vX.Y.Z" 1427 ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git;branch=master;protocol=https", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "", "") 1428 : "1.4.0", 1429 # combination version pattern 1430 ("sysprof", "git://git.yoctoproject.org/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "", "") 1431 : "1.2.0", 1432 ("u-boot-mkimage", "git://source.denx.de/u-boot/u-boot.git;branch=master;protocol=https", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "", "") 1433 : "2014.01", 1434 # version pattern "yyyymmdd" 1435 ("mobile-broadband-provider-info", "git://git.yoctoproject.org/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "", "") 1436 : "20120614", 1437 # packages with a valid UPSTREAM_CHECK_GITTAGREGEX 1438 # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing 1439 ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap;branch=master;protocol=https", "ae0394e687f1a77e966cf72f895da91840dffb8f", r"(?P<pver>(\d+\.(\d\.?)*))", "") 1440 : "0.4.3", 1441 ("build-appliance-image", "git://git.yoctoproject.org/poky;branch=master;protocol=https", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P<pver>(([0-9][\.|_]?)+[0-9]))", "") 1442 : "11.0.0", 1443 ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))", "") 1444 : "1.3.59", 1445 ("remake", "git://github.com/rocky/remake.git;protocol=https;branch=master", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))", "") 1446 : "3.82+dbg0.9", 1447 ("sysdig", "git://github.com/draios/sysdig.git;branch=dev;protocol=https", "4fb6288275f567f63515df0ff0a6518043ecfa9b", r"^(?P<pver>\d+(\.\d+)+)", "10.0.0") 1448 : "0.28.0", 1449 } 1450 1451 test_wget_uris = { 1452 # 1453 # packages with versions inside directory name 1454 # 1455 # http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2 1456 ("util-linux", "/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2", "", "") 1457 : "2.24.2", 1458 # http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz 1459 ("enchant", "/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz", "", "") 1460 : "1.6.0", 1461 # http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz 1462 ("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz", "", "") 1463 : "2.8.12.1", 1464 # https://download.gnome.org/sources/libxml2/2.9/libxml2-2.9.14.tar.xz 1465 ("libxml2", "/software/libxml2/2.9/libxml2-2.9.14.tar.xz", "", "") 1466 : "2.10.3", 1467 # 1468 # packages with versions only in current directory 1469 # 1470 # https://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2 1471 ("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "") 1472 : "2.19", 1473 # https://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2 1474 ("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "") 1475 : "20120814", 1476 # 1477 # packages with "99" in the name of possible version 1478 # 1479 # http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz 1480 ("pulseaudio", "/software/pulseaudio/releases/pulseaudio-4.0.tar.xz", "", "") 1481 : "5.0", 1482 # http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2 1483 ("xserver-xorg", "/releases/individual/xserver/xorg-server-1.15.1.tar.bz2", "", "") 1484 : "1.15.1", 1485 # 1486 # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX 1487 # 1488 # http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2 1489 # https://github.com/apple/cups/releases 1490 ("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", "/apple/cups/releases", r"(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz") 1491 : "2.0.0", 1492 # http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz 1493 # http://ftp.debian.org/debian/pool/main/d/db5.3/ 1494 ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", r"(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz") 1495 : "5.3.10", 1496 # 1497 # packages where the tarball compression changed in the new version 1498 # 1499 # http://ftp.debian.org/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz 1500 ("minicom", "/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz", "", "") 1501 : "2.8", 1502 } 1503 1504 test_crate_uris = { 1505 # basic example; version pattern "A.B.C+cargo-D.E.F" 1506 ("cargo-c", "crate://crates.io/cargo-c/0.9.18+cargo-0.69") 1507 : "0.9.29" 1508 } 1509 1510 @skipIfNoNetwork() 1511 def test_git_latest_versionstring(self): 1512 for k, v in self.test_git_uris.items(): 1513 self.d.setVar("PN", k[0]) 1514 self.d.setVar("SRCREV", k[2]) 1515 self.d.setVar("UPSTREAM_CHECK_GITTAGREGEX", k[3]) 1516 ud = bb.fetch2.FetchData(k[1], self.d) 1517 pupver= ud.method.latest_versionstring(ud, self.d) 1518 verstring = pupver[0] 1519 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) 1520 r = bb.utils.vercmp_string(v, verstring) 1521 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) 1522 if k[4]: 1523 r = bb.utils.vercmp_string(verstring, k[4]) 1524 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], verstring, k[4])) 1525 1526 def test_wget_latest_versionstring(self): 1527 testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata" 1528 server = HTTPService(testdata, host="127.0.0.1") 1529 server.start() 1530 port = server.port 1531 try: 1532 for k, v in self.test_wget_uris.items(): 1533 self.d.setVar("PN", k[0]) 1534 checkuri = "" 1535 if k[2]: 1536 checkuri = "http://127.0.0.1:%s/" % port + k[2] 1537 self.d.setVar("UPSTREAM_CHECK_URI", checkuri) 1538 self.d.setVar("UPSTREAM_CHECK_REGEX", k[3]) 1539 url = "http://127.0.0.1:%s/" % port + k[1] 1540 ud = bb.fetch2.FetchData(url, self.d) 1541 pupver = ud.method.latest_versionstring(ud, self.d) 1542 verstring = pupver[0] 1543 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) 1544 r = bb.utils.vercmp_string(v, verstring) 1545 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) 1546 finally: 1547 server.stop() 1548 1549 @skipIfNoNetwork() 1550 def test_crate_latest_versionstring(self): 1551 for k, v in self.test_crate_uris.items(): 1552 self.d.setVar("PN", k[0]) 1553 ud = bb.fetch2.FetchData(k[1], self.d) 1554 pupver = ud.method.latest_versionstring(ud, self.d) 1555 verstring = pupver[0] 1556 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) 1557 r = bb.utils.vercmp_string(v, verstring) 1558 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) 1559 1560class FetchCheckStatusTest(FetcherTest): 1561 test_wget_uris = ["https://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz", 1562 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz", 1563 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz", 1564 "https://yoctoproject.org/", 1565 "https://docs.yoctoproject.org", 1566 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz", 1567 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz", 1568 "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz", 1569 # GitHub releases are hosted on Amazon S3, which doesn't support HEAD 1570 "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz" 1571 ] 1572 1573 @skipIfNoNetwork() 1574 def test_wget_checkstatus(self): 1575 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d) 1576 for u in self.test_wget_uris: 1577 with self.subTest(url=u): 1578 ud = fetch.ud[u] 1579 m = ud.method 1580 ret = m.checkstatus(fetch, ud, self.d) 1581 self.assertTrue(ret, msg="URI %s, can't check status" % (u)) 1582 1583 @skipIfNoNetwork() 1584 def test_wget_checkstatus_connection_cache(self): 1585 from bb.fetch2 import FetchConnectionCache 1586 1587 connection_cache = FetchConnectionCache() 1588 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d, 1589 connection_cache = connection_cache) 1590 1591 for u in self.test_wget_uris: 1592 with self.subTest(url=u): 1593 ud = fetch.ud[u] 1594 m = ud.method 1595 ret = m.checkstatus(fetch, ud, self.d) 1596 self.assertTrue(ret, msg="URI %s, can't check status" % (u)) 1597 1598 connection_cache.close_connections() 1599 1600 1601class GitMakeShallowTest(FetcherTest): 1602 def setUp(self): 1603 FetcherTest.setUp(self) 1604 self.gitdir = os.path.join(self.tempdir, 'gitshallow') 1605 bb.utils.mkdirhier(self.gitdir) 1606 self.git_init() 1607 1608 def assertRefs(self, expected_refs): 1609 actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines() 1610 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs).splitlines() 1611 self.assertEqual(sorted(full_expected), sorted(actual_refs)) 1612 1613 def assertRevCount(self, expected_count, args=None): 1614 if args is None: 1615 args = ['HEAD'] 1616 revs = self.git(['rev-list'] + args) 1617 actual_count = len(revs.splitlines()) 1618 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count)) 1619 1620 def make_shallow(self, args=None): 1621 if args is None: 1622 args = ['HEAD'] 1623 return bb.process.run([bb.fetch2.git.Git.make_shallow_path] + args, cwd=self.gitdir) 1624 1625 def add_empty_file(self, path, msg=None): 1626 if msg is None: 1627 msg = path 1628 open(os.path.join(self.gitdir, path), 'w').close() 1629 self.git(['add', path]) 1630 self.git(['commit', '-m', msg, path]) 1631 1632 def test_make_shallow_single_branch_no_merge(self): 1633 self.add_empty_file('a') 1634 self.add_empty_file('b') 1635 self.assertRevCount(2) 1636 self.make_shallow() 1637 self.assertRevCount(1) 1638 1639 def test_make_shallow_single_branch_one_merge(self): 1640 self.add_empty_file('a') 1641 self.add_empty_file('b') 1642 self.git('checkout -b a_branch') 1643 self.add_empty_file('c') 1644 self.git('checkout master') 1645 self.add_empty_file('d') 1646 self.git('merge --no-ff --no-edit a_branch') 1647 self.git('branch -d a_branch') 1648 self.add_empty_file('e') 1649 self.assertRevCount(6) 1650 self.make_shallow(['HEAD~2']) 1651 self.assertRevCount(5) 1652 1653 def test_make_shallow_at_merge(self): 1654 self.add_empty_file('a') 1655 self.git('checkout -b a_branch') 1656 self.add_empty_file('b') 1657 self.git('checkout master') 1658 self.git('merge --no-ff --no-edit a_branch') 1659 self.git('branch -d a_branch') 1660 self.assertRevCount(3) 1661 self.make_shallow() 1662 self.assertRevCount(1) 1663 1664 def test_make_shallow_annotated_tag(self): 1665 self.add_empty_file('a') 1666 self.add_empty_file('b') 1667 self.git('tag -a -m a_tag a_tag') 1668 self.assertRevCount(2) 1669 self.make_shallow(['a_tag']) 1670 self.assertRevCount(1) 1671 1672 def test_make_shallow_multi_ref(self): 1673 self.add_empty_file('a') 1674 self.add_empty_file('b') 1675 self.git('checkout -b a_branch') 1676 self.add_empty_file('c') 1677 self.git('checkout master') 1678 self.add_empty_file('d') 1679 self.git('checkout -b a_branch_2') 1680 self.add_empty_file('a_tag') 1681 self.git('tag a_tag') 1682 self.git('checkout master') 1683 self.git('branch -D a_branch_2') 1684 self.add_empty_file('e') 1685 self.assertRevCount(6, ['--all']) 1686 self.make_shallow() 1687 self.assertRevCount(5, ['--all']) 1688 1689 def test_make_shallow_multi_ref_trim(self): 1690 self.add_empty_file('a') 1691 self.git('checkout -b a_branch') 1692 self.add_empty_file('c') 1693 self.git('checkout master') 1694 self.assertRevCount(1) 1695 self.assertRevCount(2, ['--all']) 1696 self.assertRefs(['master', 'a_branch']) 1697 self.make_shallow(['-r', 'master', 'HEAD']) 1698 self.assertRevCount(1, ['--all']) 1699 self.assertRefs(['master']) 1700 1701 def test_make_shallow_noop(self): 1702 self.add_empty_file('a') 1703 self.assertRevCount(1) 1704 self.make_shallow() 1705 self.assertRevCount(1) 1706 1707 @skipIfNoNetwork() 1708 def test_make_shallow_bitbake(self): 1709 self.git('remote add origin https://github.com/openembedded/bitbake') 1710 self.git('fetch --tags origin') 1711 orig_revs = len(self.git('rev-list --all').splitlines()) 1712 self.make_shallow(['refs/tags/1.10.0']) 1713 self.assertRevCount(orig_revs - 1746, ['--all']) 1714 1715class GitShallowTest(FetcherTest): 1716 def setUp(self): 1717 FetcherTest.setUp(self) 1718 self.gitdir = os.path.join(self.tempdir, 'git') 1719 self.srcdir = os.path.join(self.tempdir, 'gitsource') 1720 1721 bb.utils.mkdirhier(self.srcdir) 1722 self.git_init(cwd=self.srcdir) 1723 self.d.setVar('WORKDIR', self.tempdir) 1724 self.d.setVar('S', self.gitdir) 1725 self.d.delVar('PREMIRRORS') 1726 self.d.delVar('MIRRORS') 1727 1728 uri = 'git://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 1729 self.d.setVar('SRC_URI', uri) 1730 self.d.setVar('SRCREV', '${AUTOREV}') 1731 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') 1732 1733 self.d.setVar('BB_GIT_SHALLOW', '1') 1734 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0') 1735 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 1736 self.d.setVar("__BBSRCREV_SEEN", "1") 1737 1738 def assertRefs(self, expected_refs, cwd=None): 1739 if cwd is None: 1740 cwd = self.gitdir 1741 actual_refs = self.git(['for-each-ref', '--format=%(refname)'], cwd=cwd).splitlines() 1742 # Resolve references into the same format as the comparision (needed by git 2.48 onwards) 1743 actual_refs = self.git(['rev-parse', '--symbolic-full-name'] + actual_refs, cwd=cwd).splitlines() 1744 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs, cwd=cwd).splitlines() 1745 self.assertEqual(sorted(set(full_expected)), sorted(set(actual_refs))) 1746 1747 def assertRevCount(self, expected_count, args=None, cwd=None): 1748 if args is None: 1749 args = ['HEAD'] 1750 if cwd is None: 1751 cwd = self.gitdir 1752 revs = self.git(['rev-list'] + args, cwd=cwd) 1753 actual_count = len(revs.splitlines()) 1754 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count)) 1755 1756 def add_empty_file(self, path, cwd=None, msg=None): 1757 if msg is None: 1758 msg = path 1759 if cwd is None: 1760 cwd = self.srcdir 1761 open(os.path.join(cwd, path), 'w').close() 1762 self.git(['add', path], cwd) 1763 self.git(['commit', '-m', msg, path], cwd) 1764 1765 def fetch(self, uri=None): 1766 if uri is None: 1767 uris = self.d.getVar('SRC_URI').split() 1768 uri = uris[0] 1769 d = self.d 1770 else: 1771 d = self.d.createCopy() 1772 d.setVar('SRC_URI', uri) 1773 uri = d.expand(uri) 1774 uris = [uri] 1775 1776 fetcher = bb.fetch2.Fetch(uris, d) 1777 fetcher.download() 1778 ud = fetcher.ud[uri] 1779 return fetcher, ud 1780 1781 def fetch_and_unpack(self, uri=None): 1782 fetcher, ud = self.fetch(uri) 1783 fetcher.unpack(self.d.getVar('WORKDIR')) 1784 assert os.path.exists(self.d.getVar('S')) 1785 return fetcher, ud 1786 1787 def fetch_shallow(self, uri=None, disabled=False, keepclone=False): 1788 """Fetch a uri, generating a shallow tarball, then unpack using it""" 1789 fetcher, ud = self.fetch_and_unpack(uri) 1790 assert os.path.exists(ud.clonedir), 'Git clone in DLDIR (%s) does not exist for uri %s' % (ud.clonedir, uri) 1791 1792 # Confirm that the unpacked repo is unshallow 1793 if not disabled: 1794 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0])) 1795 1796 # fetch and unpack, from the shallow tarball 1797 bb.utils.remove(self.gitdir, recurse=True) 1798 bb.process.run('chmod u+w -R "%s"' % ud.clonedir) 1799 bb.utils.remove(ud.clonedir, recurse=True) 1800 bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True) 1801 1802 # confirm that the unpacked repo is used when no git clone or git 1803 # mirror tarball is available 1804 fetcher, ud = self.fetch_and_unpack(uri) 1805 if not disabled: 1806 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is not shallow' % self.gitdir 1807 else: 1808 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is shallow' % self.gitdir 1809 return fetcher, ud 1810 1811 def test_shallow_disabled(self): 1812 self.add_empty_file('a') 1813 self.add_empty_file('b') 1814 self.assertRevCount(2, cwd=self.srcdir) 1815 1816 self.d.setVar('BB_GIT_SHALLOW', '0') 1817 self.fetch_shallow(disabled=True) 1818 self.assertRevCount(2) 1819 1820 def test_shallow_nobranch(self): 1821 self.add_empty_file('a') 1822 self.add_empty_file('b') 1823 self.assertRevCount(2, cwd=self.srcdir) 1824 1825 srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip() 1826 self.d.setVar('SRCREV', srcrev) 1827 uri = self.d.getVar('SRC_URI').split()[0] 1828 uri = '%s;nobranch=1;bare=1' % uri 1829 1830 self.fetch_shallow(uri) 1831 self.assertRevCount(1) 1832 1833 # shallow refs are used to ensure the srcrev sticks around when we 1834 # have no other branches referencing it 1835 self.assertRefs(['refs/shallow/default']) 1836 1837 def test_shallow_default_depth_1(self): 1838 # Create initial git repo 1839 self.add_empty_file('a') 1840 self.add_empty_file('b') 1841 self.assertRevCount(2, cwd=self.srcdir) 1842 1843 self.fetch_shallow() 1844 self.assertRevCount(1) 1845 1846 def test_shallow_depth_0_disables(self): 1847 self.add_empty_file('a') 1848 self.add_empty_file('b') 1849 self.assertRevCount(2, cwd=self.srcdir) 1850 1851 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 1852 self.fetch_shallow(disabled=True) 1853 self.assertRevCount(2) 1854 1855 def test_shallow_depth_default_override(self): 1856 self.add_empty_file('a') 1857 self.add_empty_file('b') 1858 self.assertRevCount(2, cwd=self.srcdir) 1859 1860 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '2') 1861 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '1') 1862 self.fetch_shallow() 1863 self.assertRevCount(1) 1864 1865 def test_shallow_depth_default_override_disable(self): 1866 self.add_empty_file('a') 1867 self.add_empty_file('b') 1868 self.add_empty_file('c') 1869 self.assertRevCount(3, cwd=self.srcdir) 1870 1871 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 1872 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '2') 1873 self.fetch_shallow() 1874 self.assertRevCount(2) 1875 1876 def test_current_shallow_out_of_date_clone(self): 1877 # Create initial git repo 1878 self.add_empty_file('a') 1879 self.add_empty_file('b') 1880 self.add_empty_file('c') 1881 self.assertRevCount(3, cwd=self.srcdir) 1882 1883 # Clone and generate mirror tarball 1884 fetcher, ud = self.fetch() 1885 1886 # Ensure we have a current mirror tarball, but an out of date clone 1887 self.git('update-ref refs/heads/master refs/heads/master~1', cwd=ud.clonedir) 1888 self.assertRevCount(2, cwd=ud.clonedir) 1889 1890 # Fetch and unpack, from the current tarball, not the out of date clone 1891 bb.utils.remove(self.gitdir, recurse=True) 1892 fetcher, ud = self.fetch() 1893 fetcher.unpack(self.d.getVar('WORKDIR')) 1894 self.assertRevCount(1) 1895 1896 def test_shallow_single_branch_no_merge(self): 1897 self.add_empty_file('a') 1898 self.add_empty_file('b') 1899 self.assertRevCount(2, cwd=self.srcdir) 1900 1901 self.fetch_shallow() 1902 self.assertRevCount(1) 1903 assert os.path.exists(os.path.join(self.gitdir, 'a')) 1904 assert os.path.exists(os.path.join(self.gitdir, 'b')) 1905 1906 def test_shallow_no_dangling(self): 1907 self.add_empty_file('a') 1908 self.add_empty_file('b') 1909 self.assertRevCount(2, cwd=self.srcdir) 1910 1911 self.fetch_shallow() 1912 self.assertRevCount(1) 1913 assert not self.git('fsck --dangling') 1914 1915 def test_shallow_srcrev_branch_truncation(self): 1916 self.add_empty_file('a') 1917 self.add_empty_file('b') 1918 b_commit = self.git('rev-parse HEAD', cwd=self.srcdir).rstrip() 1919 self.add_empty_file('c') 1920 self.assertRevCount(3, cwd=self.srcdir) 1921 1922 self.d.setVar('SRCREV', b_commit) 1923 self.fetch_shallow() 1924 1925 # The 'c' commit was removed entirely, and 'a' was removed from history 1926 self.assertRevCount(1, ['--all']) 1927 self.assertEqual(self.git('rev-parse HEAD').strip(), b_commit) 1928 assert os.path.exists(os.path.join(self.gitdir, 'a')) 1929 assert os.path.exists(os.path.join(self.gitdir, 'b')) 1930 assert not os.path.exists(os.path.join(self.gitdir, 'c')) 1931 1932 def test_shallow_ref_pruning(self): 1933 self.add_empty_file('a') 1934 self.add_empty_file('b') 1935 self.git('branch a_branch', cwd=self.srcdir) 1936 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir) 1937 self.assertRevCount(2, cwd=self.srcdir) 1938 1939 self.fetch_shallow() 1940 1941 self.assertRefs(['master', 'origin/master']) 1942 self.assertRevCount(1) 1943 1944 def test_shallow_submodules(self): 1945 self.add_empty_file('a') 1946 self.add_empty_file('b') 1947 1948 smdir = os.path.join(self.tempdir, 'gitsubmodule') 1949 bb.utils.mkdirhier(smdir) 1950 self.git_init(cwd=smdir) 1951 # Make this look like it was cloned from a remote... 1952 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) 1953 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) 1954 self.add_empty_file('asub', cwd=smdir) 1955 self.add_empty_file('bsub', cwd=smdir) 1956 1957 self.git('submodule init', cwd=self.srcdir) 1958 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir) 1959 self.git('submodule update', cwd=self.srcdir) 1960 self.git('commit -m submodule -a', cwd=self.srcdir) 1961 1962 uri = 'gitsm://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 1963 fetcher, ud = self.fetch_shallow(uri) 1964 1965 # Verify the main repository is shallow 1966 self.assertRevCount(1) 1967 1968 # Verify the gitsubmodule directory is present 1969 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule')) 1970 1971 # Verify the submodule is also shallow 1972 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule')) 1973 1974 def test_shallow_submodule_mirrors(self): 1975 self.add_empty_file('a') 1976 self.add_empty_file('b') 1977 1978 smdir = os.path.join(self.tempdir, 'gitsubmodule') 1979 bb.utils.mkdirhier(smdir) 1980 self.git_init(cwd=smdir) 1981 # Make this look like it was cloned from a remote... 1982 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) 1983 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) 1984 self.add_empty_file('asub', cwd=smdir) 1985 self.add_empty_file('bsub', cwd=smdir) 1986 1987 self.git('submodule init', cwd=self.srcdir) 1988 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir) 1989 self.git('submodule update', cwd=self.srcdir) 1990 self.git('commit -m submodule -a', cwd=self.srcdir) 1991 1992 uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir 1993 1994 # Fetch once to generate the shallow tarball 1995 fetcher, ud = self.fetch(uri) 1996 1997 # Set up the mirror 1998 mirrordir = os.path.join(self.tempdir, 'mirror') 1999 bb.utils.rename(self.dldir, mirrordir) 2000 self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/' % mirrordir) 2001 2002 # Fetch from the mirror 2003 bb.utils.remove(self.dldir, recurse=True) 2004 bb.utils.remove(self.gitdir, recurse=True) 2005 self.fetch_and_unpack(uri) 2006 2007 # Verify the main repository is shallow 2008 self.assertRevCount(1) 2009 2010 # Verify the gitsubmodule directory is present 2011 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule')) 2012 2013 # Verify the submodule is also shallow 2014 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule')) 2015 2016 if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')): 2017 def test_shallow_annex(self): 2018 self.add_empty_file('a') 2019 self.add_empty_file('b') 2020 self.git('annex init', cwd=self.srcdir) 2021 open(os.path.join(self.srcdir, 'c'), 'w').close() 2022 self.git('annex add c', cwd=self.srcdir) 2023 self.git('commit --author "Foo Bar <foo@bar>" -m annex-c -a', cwd=self.srcdir) 2024 bb.process.run('chmod u+w -R %s' % self.srcdir) 2025 2026 uri = 'gitannex://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 2027 fetcher, ud = self.fetch_shallow(uri) 2028 2029 self.assertRevCount(1) 2030 assert './.git/annex/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0] 2031 assert os.path.exists(os.path.join(self.gitdir, 'c')) 2032 2033 def test_shallow_multi_one_uri(self): 2034 # Create initial git repo 2035 self.add_empty_file('a') 2036 self.add_empty_file('b') 2037 self.git('checkout -b a_branch', cwd=self.srcdir) 2038 self.add_empty_file('c') 2039 self.git('tag v0.0 HEAD', cwd=self.srcdir) 2040 self.add_empty_file('d') 2041 self.git('checkout master', cwd=self.srcdir) 2042 self.add_empty_file('e') 2043 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 2044 self.add_empty_file('f') 2045 self.assertRevCount(7, cwd=self.srcdir) 2046 2047 uri = self.d.getVar('SRC_URI').split()[0] 2048 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri 2049 2050 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2051 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2052 self.d.setVar('SRCREV_master', '${AUTOREV}') 2053 self.d.setVar('SRCREV_a_branch', '${AUTOREV}') 2054 2055 self.fetch_shallow(uri) 2056 2057 self.assertRevCount(4) 2058 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 2059 2060 def test_shallow_multi_one_uri_depths(self): 2061 # Create initial git repo 2062 self.add_empty_file('a') 2063 self.add_empty_file('b') 2064 self.git('checkout -b a_branch', cwd=self.srcdir) 2065 self.add_empty_file('c') 2066 self.add_empty_file('d') 2067 self.git('checkout master', cwd=self.srcdir) 2068 self.add_empty_file('e') 2069 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 2070 self.add_empty_file('f') 2071 self.assertRevCount(7, cwd=self.srcdir) 2072 2073 uri = self.d.getVar('SRC_URI').split()[0] 2074 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri 2075 2076 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2077 self.d.setVar('BB_GIT_SHALLOW_DEPTH_master', '3') 2078 self.d.setVar('BB_GIT_SHALLOW_DEPTH_a_branch', '1') 2079 self.d.setVar('SRCREV_master', '${AUTOREV}') 2080 self.d.setVar('SRCREV_a_branch', '${AUTOREV}') 2081 2082 self.fetch_shallow(uri) 2083 2084 self.assertRevCount(4, ['--all']) 2085 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 2086 2087 def test_shallow_clone_preferred_over_shallow(self): 2088 self.add_empty_file('a') 2089 self.add_empty_file('b') 2090 2091 # Fetch once to generate the shallow tarball 2092 fetcher, ud = self.fetch() 2093 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0])) 2094 2095 # Fetch and unpack with both the clonedir and shallow tarball available 2096 bb.utils.remove(self.gitdir, recurse=True) 2097 fetcher, ud = self.fetch_and_unpack() 2098 2099 # The unpacked tree should *not* be shallow 2100 self.assertRevCount(2) 2101 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')) 2102 2103 def test_shallow_mirrors(self): 2104 self.add_empty_file('a') 2105 self.add_empty_file('b') 2106 2107 # Fetch once to generate the shallow tarball 2108 fetcher, ud = self.fetch() 2109 mirrortarball = ud.mirrortarballs[0] 2110 assert os.path.exists(os.path.join(self.dldir, mirrortarball)) 2111 2112 # Set up the mirror 2113 mirrordir = os.path.join(self.tempdir, 'mirror') 2114 bb.utils.mkdirhier(mirrordir) 2115 self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/' % mirrordir) 2116 2117 bb.utils.rename(os.path.join(self.dldir, mirrortarball), 2118 os.path.join(mirrordir, mirrortarball)) 2119 2120 # Fetch from the mirror 2121 bb.utils.remove(self.dldir, recurse=True) 2122 bb.utils.remove(self.gitdir, recurse=True) 2123 self.fetch_and_unpack() 2124 self.assertRevCount(1) 2125 2126 def test_shallow_invalid_depth(self): 2127 self.add_empty_file('a') 2128 self.add_empty_file('b') 2129 2130 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '-12') 2131 with self.assertRaises(bb.fetch2.FetchError): 2132 self.fetch() 2133 2134 def test_shallow_invalid_depth_default(self): 2135 self.add_empty_file('a') 2136 self.add_empty_file('b') 2137 2138 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '-12') 2139 with self.assertRaises(bb.fetch2.FetchError): 2140 self.fetch() 2141 2142 def test_shallow_extra_refs(self): 2143 self.add_empty_file('a') 2144 self.add_empty_file('b') 2145 self.git('branch a_branch', cwd=self.srcdir) 2146 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir) 2147 self.assertRevCount(2, cwd=self.srcdir) 2148 2149 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/a_branch') 2150 self.fetch_shallow() 2151 2152 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 2153 self.assertRevCount(1) 2154 2155 def test_shallow_extra_refs_wildcard(self): 2156 self.add_empty_file('a') 2157 self.add_empty_file('b') 2158 self.git('branch a_branch', cwd=self.srcdir) 2159 self.git('tag v1.0', cwd=self.srcdir) 2160 self.assertRefs(['master', 'a_branch', 'v1.0'], cwd=self.srcdir) 2161 self.assertRevCount(2, cwd=self.srcdir) 2162 2163 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*') 2164 self.fetch_shallow() 2165 2166 self.assertRefs(['master', 'origin/master', 'v1.0']) 2167 self.assertRevCount(1) 2168 2169 def test_shallow_missing_extra_refs(self): 2170 self.add_empty_file('a') 2171 self.add_empty_file('b') 2172 2173 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/foo') 2174 with self.assertRaises(bb.fetch2.FetchError): 2175 self.fetch() 2176 2177 def test_shallow_missing_extra_refs_wildcard(self): 2178 self.add_empty_file('a') 2179 self.add_empty_file('b') 2180 2181 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*') 2182 self.fetch() 2183 2184 def test_shallow_remove_revs(self): 2185 # Create initial git repo 2186 self.add_empty_file('a') 2187 self.add_empty_file('b') 2188 self.git('checkout -b a_branch', cwd=self.srcdir) 2189 self.add_empty_file('c') 2190 self.add_empty_file('d') 2191 self.git('checkout master', cwd=self.srcdir) 2192 self.git('tag v0.0 a_branch', cwd=self.srcdir) 2193 self.add_empty_file('e') 2194 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 2195 self.git('branch -d a_branch', cwd=self.srcdir) 2196 self.add_empty_file('f') 2197 self.assertRevCount(7, cwd=self.srcdir) 2198 2199 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2200 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2201 2202 self.fetch_shallow() 2203 2204 self.assertRevCount(2) 2205 2206 def test_shallow_invalid_revs(self): 2207 self.add_empty_file('a') 2208 self.add_empty_file('b') 2209 2210 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2211 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2212 2213 with self.assertRaises(bb.fetch2.FetchError): 2214 self.fetch() 2215 2216 def test_shallow_fetch_missing_revs(self): 2217 self.add_empty_file('a') 2218 self.add_empty_file('b') 2219 fetcher, ud = self.fetch(self.d.getVar('SRC_URI')) 2220 self.git('tag v0.0 master', cwd=self.srcdir) 2221 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2222 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2223 2224 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm: 2225 self.fetch_shallow() 2226 self.assertIn("fatal: no commits selected for shallow requests", cm.output[0]) 2227 2228 def test_shallow_fetch_missing_revs_fails(self): 2229 self.add_empty_file('a') 2230 self.add_empty_file('b') 2231 fetcher, ud = self.fetch(self.d.getVar('SRC_URI')) 2232 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2233 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2234 2235 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm: 2236 self.fetch_shallow() 2237 self.assertIn("Unable to find revision v0.0 even from upstream", cm.output[0]) 2238 2239 @skipIfNoNetwork() 2240 def test_bitbake(self): 2241 self.git('remote add --mirror=fetch origin https://github.com/openembedded/bitbake', cwd=self.srcdir) 2242 self.git('config core.bare true', cwd=self.srcdir) 2243 self.git('fetch', cwd=self.srcdir) 2244 2245 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2246 # Note that the 1.10.0 tag is annotated, so this also tests 2247 # reference of an annotated vs unannotated tag 2248 self.d.setVar('BB_GIT_SHALLOW_REVS', '1.10.0') 2249 2250 self.fetch_shallow() 2251 2252 # Confirm that the history of 1.10.0 was removed 2253 orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines()) 2254 revs = len(self.git('rev-list master').splitlines()) 2255 self.assertNotEqual(orig_revs, revs) 2256 self.assertRefs(['master', 'origin/master']) 2257 self.assertRevCount(orig_revs - 1760) 2258 2259 def test_that_unpack_throws_an_error_when_the_git_clone_nor_shallow_tarball_exist(self): 2260 self.add_empty_file('a') 2261 fetcher, ud = self.fetch() 2262 bb.utils.remove(self.gitdir, recurse=True) 2263 bb.utils.remove(self.dldir, recurse=True) 2264 2265 with self.assertRaises(bb.fetch2.UnpackError) as context: 2266 fetcher.unpack(self.d.getVar('WORKDIR')) 2267 2268 self.assertIn("No up to date source found", context.exception.msg) 2269 self.assertIn("clone directory not available or not up to date", context.exception.msg) 2270 2271 @skipIfNoNetwork() 2272 def test_that_unpack_does_work_when_using_git_shallow_tarball_but_tarball_is_not_available(self): 2273 self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0') 2274 self.d.setVar('BB_GIT_SHALLOW', '1') 2275 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 2276 fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests;branch=master;protocol=https"], self.d) 2277 fetcher.download() 2278 2279 bb.utils.remove(self.dldir + "/*.tar.gz") 2280 fetcher.unpack(self.unpackdir) 2281 2282 dir = os.listdir(self.unpackdir + "/git/") 2283 self.assertIn("fstests.doap", dir) 2284 2285class GitLfsTest(FetcherTest): 2286 def skipIfNoGitLFS(): 2287 import shutil 2288 if not shutil.which('git-lfs'): 2289 return unittest.skip('git-lfs not installed') 2290 return lambda f: f 2291 2292 def setUp(self): 2293 FetcherTest.setUp(self) 2294 2295 self.gitdir = os.path.join(self.tempdir, 'git') 2296 self.srcdir = os.path.join(self.tempdir, 'gitsource') 2297 2298 self.d.setVar('WORKDIR', self.tempdir) 2299 self.d.setVar('S', self.gitdir) 2300 self.d.delVar('PREMIRRORS') 2301 self.d.delVar('MIRRORS') 2302 2303 self.d.setVar('SRCREV', '${AUTOREV}') 2304 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') 2305 self.d.setVar("__BBSRCREV_SEEN", "1") 2306 2307 bb.utils.mkdirhier(self.srcdir) 2308 self.git_init(cwd=self.srcdir) 2309 self.commit_file('.gitattributes', '*.mp3 filter=lfs -text') 2310 2311 def commit_file(self, filename, content): 2312 with open(os.path.join(self.srcdir, filename), "w") as f: 2313 f.write(content) 2314 self.git(["add", filename], cwd=self.srcdir) 2315 self.git(["commit", "-m", "Change"], cwd=self.srcdir) 2316 return self.git(["rev-parse", "HEAD"], cwd=self.srcdir).strip() 2317 2318 def fetch(self, uri=None, download=True): 2319 uris = self.d.getVar('SRC_URI').split() 2320 uri = uris[0] 2321 d = self.d 2322 2323 fetcher = bb.fetch2.Fetch(uris, d) 2324 if download: 2325 fetcher.download() 2326 ud = fetcher.ud[uri] 2327 return fetcher, ud 2328 2329 def get_real_git_lfs_file(self): 2330 self.d.setVar('PATH', os.environ.get('PATH')) 2331 fetcher, ud = self.fetch() 2332 fetcher.unpack(self.d.getVar('WORKDIR')) 2333 unpacked_lfs_file = os.path.join(self.d.getVar('WORKDIR'), 'git', "Cat_poster_1.jpg") 2334 return unpacked_lfs_file 2335 2336 @skipIfNoGitLFS() 2337 def test_fetch_lfs_on_srcrev_change(self): 2338 """Test if fetch downloads missing LFS objects when a different revision within an existing repository is requested""" 2339 self.git(["lfs", "install", "--local"], cwd=self.srcdir) 2340 2341 @contextlib.contextmanager 2342 def hide_upstream_repository(): 2343 """Hide the upstream repository to make sure that git lfs cannot pull from it""" 2344 temp_name = self.srcdir + ".bak" 2345 os.rename(self.srcdir, temp_name) 2346 try: 2347 yield 2348 finally: 2349 os.rename(temp_name, self.srcdir) 2350 2351 def fetch_and_verify(revision, filename, content): 2352 self.d.setVar('SRCREV', revision) 2353 fetcher, ud = self.fetch() 2354 2355 with hide_upstream_repository(): 2356 workdir = self.d.getVar('WORKDIR') 2357 fetcher.unpack(workdir) 2358 2359 with open(os.path.join(workdir, "git", filename)) as f: 2360 self.assertEqual(f.read(), content) 2361 2362 commit_1 = self.commit_file("a.mp3", "version 1") 2363 commit_2 = self.commit_file("a.mp3", "version 2") 2364 2365 self.d.setVar('SRC_URI', "git://%s;protocol=file;lfs=1;branch=master" % self.srcdir) 2366 2367 # Seed the local download folder by fetching the latest commit and verifying that the LFS contents are 2368 # available even when the upstream repository disappears. 2369 fetch_and_verify(commit_2, "a.mp3", "version 2") 2370 # Verify that even when an older revision is fetched, the needed LFS objects are fetched into the download 2371 # folder. 2372 fetch_and_verify(commit_1, "a.mp3", "version 1") 2373 2374 @skipIfNoGitLFS() 2375 @skipIfNoNetwork() 2376 def test_real_git_lfs_repo_succeeds_without_lfs_param(self): 2377 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master") 2378 f = self.get_real_git_lfs_file() 2379 self.assertTrue(os.path.exists(f)) 2380 self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f)) 2381 2382 @skipIfNoGitLFS() 2383 @skipIfNoNetwork() 2384 def test_real_git_lfs_repo_succeeds(self): 2385 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=1") 2386 f = self.get_real_git_lfs_file() 2387 self.assertTrue(os.path.exists(f)) 2388 self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f)) 2389 2390 @skipIfNoGitLFS() 2391 @skipIfNoNetwork() 2392 def test_real_git_lfs_repo_skips(self): 2393 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=0") 2394 f = self.get_real_git_lfs_file() 2395 # This is the actual non-smudged placeholder file on the repo if git-lfs does not run 2396 lfs_file = ( 2397 'version https://git-lfs.github.com/spec/v1\n' 2398 'oid sha256:34be66b1a39a1955b46a12588df9d5f6fc1da790e05cf01f3c7422f4bbbdc26b\n' 2399 'size 11423554\n' 2400 ) 2401 2402 with open(f) as fh: 2403 self.assertEqual(lfs_file, fh.read()) 2404 2405 @skipIfNoGitLFS() 2406 def test_lfs_enabled(self): 2407 import shutil 2408 2409 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir 2410 self.d.setVar('SRC_URI', uri) 2411 2412 # With git-lfs installed, test that we can fetch and unpack 2413 fetcher, ud = self.fetch() 2414 shutil.rmtree(self.gitdir, ignore_errors=True) 2415 fetcher.unpack(self.d.getVar('WORKDIR')) 2416 2417 @skipIfNoGitLFS() 2418 def test_lfs_disabled(self): 2419 import shutil 2420 2421 uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir 2422 self.d.setVar('SRC_URI', uri) 2423 2424 # Verify that the fetcher can survive even if the source 2425 # repository has Git LFS usage configured. 2426 fetcher, ud = self.fetch() 2427 fetcher.unpack(self.d.getVar('WORKDIR')) 2428 2429 def test_lfs_enabled_not_installed(self): 2430 import shutil 2431 2432 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir 2433 self.d.setVar('SRC_URI', uri) 2434 2435 # Careful: suppress initial attempt at downloading 2436 fetcher, ud = self.fetch(uri=None, download=False) 2437 2438 # Artificially assert that git-lfs is not installed, so 2439 # we can verify a failure to unpack in it's absence. 2440 old_find_git_lfs = ud.method._find_git_lfs 2441 try: 2442 # If git-lfs cannot be found, the unpack should throw an error 2443 with self.assertRaises(bb.fetch2.FetchError): 2444 fetcher.download() 2445 ud.method._find_git_lfs = lambda d: False 2446 shutil.rmtree(self.gitdir, ignore_errors=True) 2447 fetcher.unpack(self.d.getVar('WORKDIR')) 2448 finally: 2449 ud.method._find_git_lfs = old_find_git_lfs 2450 2451 def test_lfs_disabled_not_installed(self): 2452 import shutil 2453 2454 uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir 2455 self.d.setVar('SRC_URI', uri) 2456 2457 # Careful: suppress initial attempt at downloading 2458 fetcher, ud = self.fetch(uri=None, download=False) 2459 2460 # Artificially assert that git-lfs is not installed, so 2461 # we can verify a failure to unpack in it's absence. 2462 old_find_git_lfs = ud.method._find_git_lfs 2463 try: 2464 # Even if git-lfs cannot be found, the unpack should be successful 2465 fetcher.download() 2466 ud.method._find_git_lfs = lambda d: False 2467 shutil.rmtree(self.gitdir, ignore_errors=True) 2468 fetcher.unpack(self.d.getVar('WORKDIR')) 2469 finally: 2470 ud.method._find_git_lfs = old_find_git_lfs 2471 2472class GitURLWithSpacesTest(FetcherTest): 2473 test_git_urls = { 2474 "git://tfs-example.org:22/tfs/example%20path/example.git;branch=master" : { 2475 'url': 'git://tfs-example.org:22/tfs/example%20path/example.git;branch=master', 2476 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example.git', 2477 'path': '/tfs/example path/example.git' 2478 }, 2479 "git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master" : { 2480 'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master', 2481 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example_repo.git', 2482 'path': '/tfs/example path/example repo.git' 2483 } 2484 } 2485 2486 def test_urls(self): 2487 2488 # Set fake SRCREV to stop git fetcher from trying to contact non-existent git repo 2489 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 2490 2491 for test_git_url, ref in self.test_git_urls.items(): 2492 2493 fetcher = bb.fetch.Fetch([test_git_url], self.d) 2494 ud = fetcher.ud[fetcher.urls[0]] 2495 2496 self.assertEqual(ud.url, ref['url']) 2497 self.assertEqual(ud.path, ref['path']) 2498 self.assertEqual(ud.localfile, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2499 self.assertEqual(ud.localpath, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2500 self.assertEqual(ud.lockfile, os.path.join(self.dldir, "git2", ref['gitsrcname'] + '.lock')) 2501 self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2502 self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz')) 2503 2504class CrateTest(FetcherTest): 2505 @skipIfNoNetwork() 2506 def test_crate_url(self): 2507 2508 uri = "crate://crates.io/glob/0.2.11" 2509 self.d.setVar('SRC_URI', uri) 2510 2511 uris = self.d.getVar('SRC_URI').split() 2512 d = self.d 2513 2514 fetcher = bb.fetch2.Fetch(uris, self.d) 2515 ud = fetcher.ud[fetcher.urls[0]] 2516 2517 self.assertIn("name", ud.parm) 2518 self.assertEqual(ud.parm["name"], "glob-0.2.11") 2519 self.assertIn("downloadfilename", ud.parm) 2520 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") 2521 2522 fetcher.download() 2523 fetcher.unpack(self.tempdir) 2524 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2525 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done']) 2526 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) 2527 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) 2528 2529 @skipIfNoNetwork() 2530 def test_crate_url_matching_recipe(self): 2531 2532 self.d.setVar('BP', 'glob-0.2.11') 2533 2534 uri = "crate://crates.io/glob/0.2.11" 2535 self.d.setVar('SRC_URI', uri) 2536 2537 uris = self.d.getVar('SRC_URI').split() 2538 d = self.d 2539 2540 fetcher = bb.fetch2.Fetch(uris, self.d) 2541 ud = fetcher.ud[fetcher.urls[0]] 2542 2543 self.assertIn("name", ud.parm) 2544 self.assertEqual(ud.parm["name"], "glob-0.2.11") 2545 self.assertIn("downloadfilename", ud.parm) 2546 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") 2547 2548 fetcher.download() 2549 fetcher.unpack(self.tempdir) 2550 self.assertEqual(sorted(os.listdir(self.tempdir)), ['download', 'glob-0.2.11', 'unpacked']) 2551 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done']) 2552 self.assertTrue(os.path.exists(self.tempdir + "/glob-0.2.11/src/lib.rs")) 2553 2554 @skipIfNoNetwork() 2555 def test_crate_url_params(self): 2556 2557 uri = "crate://crates.io/aho-corasick/0.7.20;name=aho-corasick-renamed" 2558 self.d.setVar('SRC_URI', uri) 2559 2560 uris = self.d.getVar('SRC_URI').split() 2561 d = self.d 2562 2563 fetcher = bb.fetch2.Fetch(uris, self.d) 2564 ud = fetcher.ud[fetcher.urls[0]] 2565 2566 self.assertIn("name", ud.parm) 2567 self.assertEqual(ud.parm["name"], "aho-corasick-renamed") 2568 self.assertIn("downloadfilename", ud.parm) 2569 self.assertEqual(ud.parm["downloadfilename"], "aho-corasick-0.7.20.crate") 2570 2571 fetcher.download() 2572 fetcher.unpack(self.tempdir) 2573 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2574 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['aho-corasick-0.7.20.crate', 'aho-corasick-0.7.20.crate.done']) 2575 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/.cargo-checksum.json")) 2576 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/src/lib.rs")) 2577 2578 @skipIfNoNetwork() 2579 def test_crate_url_multi(self): 2580 2581 uri = "crate://crates.io/glob/0.2.11 crate://crates.io/time/0.1.35" 2582 self.d.setVar('SRC_URI', uri) 2583 2584 uris = self.d.getVar('SRC_URI').split() 2585 d = self.d 2586 2587 fetcher = bb.fetch2.Fetch(uris, self.d) 2588 ud = fetcher.ud[fetcher.urls[0]] 2589 2590 self.assertIn("name", ud.parm) 2591 self.assertEqual(ud.parm["name"], "glob-0.2.11") 2592 self.assertIn("downloadfilename", ud.parm) 2593 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") 2594 2595 ud = fetcher.ud[fetcher.urls[1]] 2596 self.assertIn("name", ud.parm) 2597 self.assertEqual(ud.parm["name"], "time-0.1.35") 2598 self.assertIn("downloadfilename", ud.parm) 2599 self.assertEqual(ud.parm["downloadfilename"], "time-0.1.35.crate") 2600 2601 fetcher.download() 2602 fetcher.unpack(self.tempdir) 2603 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2604 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done', 'time-0.1.35.crate', 'time-0.1.35.crate.done']) 2605 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) 2606 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) 2607 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/.cargo-checksum.json")) 2608 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/src/lib.rs")) 2609 2610 @skipIfNoNetwork() 2611 def test_crate_incorrect_cksum(self): 2612 uri = "crate://crates.io/aho-corasick/0.7.20" 2613 self.d.setVar('SRC_URI', uri) 2614 self.d.setVarFlag("SRC_URI", "aho-corasick-0.7.20.sha256sum", hashlib.sha256("Invalid".encode("utf-8")).hexdigest()) 2615 2616 uris = self.d.getVar('SRC_URI').split() 2617 2618 fetcher = bb.fetch2.Fetch(uris, self.d) 2619 with self.assertRaisesRegex(bb.fetch2.FetchError, "Fetcher failure for URL"): 2620 fetcher.download() 2621 2622class NPMTest(FetcherTest): 2623 def skipIfNoNpm(): 2624 import shutil 2625 if not shutil.which('npm'): 2626 return unittest.skip('npm not installed') 2627 return lambda f: f 2628 2629 @skipIfNoNpm() 2630 @skipIfNoNetwork() 2631 def test_npm(self): 2632 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2633 fetcher = bb.fetch.Fetch(urls, self.d) 2634 ud = fetcher.ud[fetcher.urls[0]] 2635 fetcher.download() 2636 self.assertTrue(os.path.exists(ud.localpath)) 2637 self.assertTrue(os.path.exists(ud.localpath + '.done')) 2638 self.assertTrue(os.path.exists(ud.resolvefile)) 2639 fetcher.unpack(self.unpackdir) 2640 unpackdir = os.path.join(self.unpackdir, 'npm') 2641 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2642 2643 @skipIfNoNpm() 2644 @skipIfNoNetwork() 2645 def test_npm_bad_checksum(self): 2646 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2647 # Fetch once to get a tarball 2648 fetcher = bb.fetch.Fetch(urls, self.d) 2649 ud = fetcher.ud[fetcher.urls[0]] 2650 fetcher.download() 2651 self.assertTrue(os.path.exists(ud.localpath)) 2652 # Modify the tarball 2653 bad = b'bad checksum' 2654 with open(ud.localpath, 'wb') as f: 2655 f.write(bad) 2656 # Verify that the tarball is fetched again 2657 fetcher.download() 2658 badsum = hashlib.sha512(bad).hexdigest() 2659 self.assertTrue(os.path.exists(ud.localpath + '_bad-checksum_' + badsum)) 2660 self.assertTrue(os.path.exists(ud.localpath)) 2661 2662 @skipIfNoNpm() 2663 @skipIfNoNetwork() 2664 def test_npm_premirrors(self): 2665 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2666 # Fetch once to get a tarball 2667 fetcher = bb.fetch.Fetch(urls, self.d) 2668 ud = fetcher.ud[fetcher.urls[0]] 2669 fetcher.download() 2670 self.assertTrue(os.path.exists(ud.localpath)) 2671 2672 # Setup the mirror by renaming the download directory 2673 mirrordir = os.path.join(self.tempdir, 'mirror') 2674 bb.utils.rename(self.dldir, mirrordir) 2675 os.mkdir(self.dldir) 2676 2677 # Configure the premirror to be used 2678 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/npm2' % mirrordir) 2679 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2680 2681 # Fetch again 2682 self.assertFalse(os.path.exists(ud.localpath)) 2683 # The npm fetcher doesn't handle that the .resolved file disappears 2684 # while the fetcher object exists, which it does when we rename the 2685 # download directory to "mirror" above. Thus we need a new fetcher to go 2686 # with the now empty download directory. 2687 fetcher = bb.fetch.Fetch(urls, self.d) 2688 ud = fetcher.ud[fetcher.urls[0]] 2689 fetcher.download() 2690 self.assertTrue(os.path.exists(ud.localpath)) 2691 2692 @skipIfNoNpm() 2693 @skipIfNoNetwork() 2694 def test_npm_premirrors_with_specified_filename(self): 2695 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2696 # Fetch once to get a tarball 2697 fetcher = bb.fetch.Fetch(urls, self.d) 2698 ud = fetcher.ud[fetcher.urls[0]] 2699 fetcher.download() 2700 self.assertTrue(os.path.exists(ud.localpath)) 2701 # Setup the mirror 2702 mirrordir = os.path.join(self.tempdir, 'mirror') 2703 bb.utils.mkdirhier(mirrordir) 2704 mirrorfilename = os.path.join(mirrordir, os.path.basename(ud.localpath)) 2705 os.replace(ud.localpath, mirrorfilename) 2706 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s' % mirrorfilename) 2707 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2708 # Fetch again 2709 self.assertFalse(os.path.exists(ud.localpath)) 2710 fetcher.download() 2711 self.assertTrue(os.path.exists(ud.localpath)) 2712 2713 @skipIfNoNpm() 2714 @skipIfNoNetwork() 2715 def test_npm_mirrors(self): 2716 # Fetch once to get a tarball 2717 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2718 fetcher = bb.fetch.Fetch(urls, self.d) 2719 ud = fetcher.ud[fetcher.urls[0]] 2720 fetcher.download() 2721 self.assertTrue(os.path.exists(ud.localpath)) 2722 # Setup the mirror 2723 mirrordir = os.path.join(self.tempdir, 'mirror') 2724 bb.utils.mkdirhier(mirrordir) 2725 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 2726 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 2727 # Update the resolved url to an invalid url 2728 with open(ud.resolvefile, 'r') as f: 2729 url = f.read() 2730 uri = URI(url) 2731 uri.path = '/invalid' 2732 with open(ud.resolvefile, 'w') as f: 2733 f.write(str(uri)) 2734 # Fetch again 2735 self.assertFalse(os.path.exists(ud.localpath)) 2736 fetcher.download() 2737 self.assertTrue(os.path.exists(ud.localpath)) 2738 2739 @skipIfNoNpm() 2740 @skipIfNoNetwork() 2741 def test_npm_destsuffix_downloadfilename(self): 2742 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz'] 2743 fetcher = bb.fetch.Fetch(urls, self.d) 2744 fetcher.download() 2745 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'foo-bar.tgz'))) 2746 fetcher.unpack(self.unpackdir) 2747 unpackdir = os.path.join(self.unpackdir, 'foo', 'bar') 2748 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2749 2750 def test_npm_no_network_no_tarball(self): 2751 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2752 self.d.setVar('BB_NO_NETWORK', '1') 2753 fetcher = bb.fetch.Fetch(urls, self.d) 2754 with self.assertRaises(bb.fetch2.NetworkAccess): 2755 fetcher.download() 2756 2757 @skipIfNoNpm() 2758 @skipIfNoNetwork() 2759 def test_npm_no_network_with_tarball(self): 2760 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2761 # Fetch once to get a tarball 2762 fetcher = bb.fetch.Fetch(urls, self.d) 2763 fetcher.download() 2764 # Disable network access 2765 self.d.setVar('BB_NO_NETWORK', '1') 2766 # Fetch again 2767 fetcher.download() 2768 fetcher.unpack(self.unpackdir) 2769 unpackdir = os.path.join(self.unpackdir, 'npm') 2770 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2771 2772 @skipIfNoNpm() 2773 @skipIfNoNetwork() 2774 def test_npm_registry_alternate(self): 2775 urls = ['npm://skimdb.npmjs.com;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2776 fetcher = bb.fetch.Fetch(urls, self.d) 2777 fetcher.download() 2778 fetcher.unpack(self.unpackdir) 2779 unpackdir = os.path.join(self.unpackdir, 'npm') 2780 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2781 2782 @skipIfNoNpm() 2783 @skipIfNoNetwork() 2784 def test_npm_version_latest(self): 2785 url = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=latest'] 2786 fetcher = bb.fetch.Fetch(urls, self.d) 2787 fetcher.download() 2788 fetcher.unpack(self.unpackdir) 2789 unpackdir = os.path.join(self.unpackdir, 'npm') 2790 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2791 2792 @skipIfNoNpm() 2793 @skipIfNoNetwork() 2794 def test_npm_registry_invalid(self): 2795 urls = ['npm://registry.invalid.org;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2796 fetcher = bb.fetch.Fetch(urls, self.d) 2797 with self.assertRaises(bb.fetch2.FetchError): 2798 fetcher.download() 2799 2800 @skipIfNoNpm() 2801 @skipIfNoNetwork() 2802 def test_npm_package_invalid(self): 2803 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/invalid;version=1.0.0'] 2804 fetcher = bb.fetch.Fetch(urls, self.d) 2805 with self.assertRaises(bb.fetch2.FetchError): 2806 fetcher.download() 2807 2808 @skipIfNoNpm() 2809 @skipIfNoNetwork() 2810 def test_npm_version_invalid(self): 2811 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=invalid'] 2812 with self.assertRaises(bb.fetch2.ParameterError): 2813 fetcher = bb.fetch.Fetch(urls, self.d) 2814 2815 @skipIfNoNpm() 2816 @skipIfNoNetwork() 2817 def test_npm_registry_none(self): 2818 urls = ['npm://;package=@savoirfairelinux/node-server-example;version=1.0.0'] 2819 with self.assertRaises(bb.fetch2.MalformedUrl): 2820 fetcher = bb.fetch.Fetch(urls, self.d) 2821 2822 @skipIfNoNpm() 2823 @skipIfNoNetwork() 2824 def test_npm_package_none(self): 2825 urls = ['npm://registry.npmjs.org;version=1.0.0'] 2826 with self.assertRaises(bb.fetch2.MissingParameterError): 2827 fetcher = bb.fetch.Fetch(urls, self.d) 2828 2829 @skipIfNoNpm() 2830 @skipIfNoNetwork() 2831 def test_npm_version_none(self): 2832 urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example'] 2833 with self.assertRaises(bb.fetch2.MissingParameterError): 2834 fetcher = bb.fetch.Fetch(urls, self.d) 2835 2836 def create_shrinkwrap_file(self, data): 2837 import json 2838 datadir = os.path.join(self.tempdir, 'data') 2839 swfile = os.path.join(datadir, 'npm-shrinkwrap.json') 2840 bb.utils.mkdirhier(datadir) 2841 with open(swfile, 'w') as f: 2842 json.dump(data, f) 2843 return swfile 2844 2845 @skipIfNoNetwork() 2846 def test_npmsw(self): 2847 swfile = self.create_shrinkwrap_file({ 2848 'packages': { 2849 'node_modules/array-flatten': { 2850 'version': '1.1.1', 2851 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2852 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=', 2853 'dependencies': { 2854 'content-type': "1.0.4" 2855 } 2856 }, 2857 'node_modules/array-flatten/node_modules/content-type': { 2858 'version': '1.0.4', 2859 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', 2860 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', 2861 'dependencies': { 2862 'cookie': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09' 2863 } 2864 }, 2865 'node_modules/array-flatten/node_modules/content-type/node_modules/cookie': { 2866 'resolved': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09' 2867 } 2868 } 2869 }) 2870 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2871 fetcher.download() 2872 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2873 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2874 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) 2875 fetcher.unpack(self.unpackdir) 2876 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm-shrinkwrap.json'))) 2877 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'node_modules', 'array-flatten', 'package.json'))) 2878 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'package.json'))) 2879 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'node_modules', 'cookie', 'package.json'))) 2880 2881 @skipIfNoNetwork() 2882 def test_npmsw_git(self): 2883 swfile = self.create_shrinkwrap_file({ 2884 'packages': { 2885 'node_modules/cookie': { 2886 'resolved': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09' 2887 } 2888 } 2889 }) 2890 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2891 fetcher.download() 2892 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) 2893 2894 @skipIfNoNetwork() 2895 def test_npmsw_dev(self): 2896 swfile = self.create_shrinkwrap_file({ 2897 'packages': { 2898 'node_modules/array-flatten': { 2899 'version': '1.1.1', 2900 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2901 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2902 }, 2903 'node_modules/content-type': { 2904 'version': '1.0.4', 2905 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', 2906 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', 2907 'dev': True 2908 } 2909 } 2910 }) 2911 # Fetch with dev disabled 2912 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2913 fetcher.download() 2914 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2915 self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2916 # Fetch with dev enabled 2917 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';dev=1'], self.d) 2918 fetcher.download() 2919 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2920 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2921 2922 @skipIfNoNetwork() 2923 def test_npmsw_destsuffix(self): 2924 swfile = self.create_shrinkwrap_file({ 2925 'packages': { 2926 'node_modules/array-flatten': { 2927 'version': '1.1.1', 2928 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2929 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2930 } 2931 } 2932 }) 2933 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';destsuffix=foo/bar'], self.d) 2934 fetcher.download() 2935 fetcher.unpack(self.unpackdir) 2936 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'foo', 'bar', 'node_modules', 'array-flatten', 'package.json'))) 2937 2938 def test_npmsw_no_network_no_tarball(self): 2939 swfile = self.create_shrinkwrap_file({ 2940 'packages': { 2941 'node_modules/array-flatten': { 2942 'version': '1.1.1', 2943 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2944 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2945 } 2946 } 2947 }) 2948 self.d.setVar('BB_NO_NETWORK', '1') 2949 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2950 with self.assertRaises(bb.fetch2.NetworkAccess): 2951 fetcher.download() 2952 2953 @skipIfNoNpm() 2954 @skipIfNoNetwork() 2955 def test_npmsw_no_network_with_tarball(self): 2956 # Fetch once to get a tarball 2957 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 2958 fetcher.download() 2959 # Disable network access 2960 self.d.setVar('BB_NO_NETWORK', '1') 2961 # Fetch again 2962 swfile = self.create_shrinkwrap_file({ 2963 'packages': { 2964 'node_modules/array-flatten': { 2965 'version': '1.1.1', 2966 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2967 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2968 } 2969 } 2970 }) 2971 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2972 fetcher.download() 2973 fetcher.unpack(self.unpackdir) 2974 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'node_modules', 'array-flatten', 'package.json'))) 2975 2976 @skipIfNoNetwork() 2977 def test_npmsw_npm_reusability(self): 2978 # Fetch once with npmsw 2979 swfile = self.create_shrinkwrap_file({ 2980 'packages': { 2981 'node_modules/array-flatten': { 2982 'version': '1.1.1', 2983 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2984 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2985 } 2986 } 2987 }) 2988 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2989 fetcher.download() 2990 # Disable network access 2991 self.d.setVar('BB_NO_NETWORK', '1') 2992 # Fetch again with npm 2993 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 2994 fetcher.download() 2995 fetcher.unpack(self.unpackdir) 2996 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm', 'package.json'))) 2997 2998 @skipIfNoNetwork() 2999 def test_npmsw_bad_checksum(self): 3000 # Try to fetch with bad checksum 3001 swfile = self.create_shrinkwrap_file({ 3002 'packages': { 3003 'node_modules/array-flatten': { 3004 'version': '1.1.1', 3005 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3006 'integrity': 'sha1-gfNEp2hqgLTFKT6P3AsBYMgsBqg=' 3007 } 3008 } 3009 }) 3010 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3011 with self.assertRaises(bb.fetch2.FetchError): 3012 fetcher.download() 3013 # Fetch correctly to get a tarball 3014 swfile = self.create_shrinkwrap_file({ 3015 'packages': { 3016 'node_modules/array-flatten': { 3017 'version': '1.1.1', 3018 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3019 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3020 } 3021 } 3022 }) 3023 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3024 fetcher.download() 3025 localpath = os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz') 3026 self.assertTrue(os.path.exists(localpath)) 3027 # Modify the tarball 3028 bad = b'bad checksum' 3029 with open(localpath, 'wb') as f: 3030 f.write(bad) 3031 # Verify that the tarball is fetched again 3032 fetcher.download() 3033 badsum = hashlib.sha1(bad).hexdigest() 3034 self.assertTrue(os.path.exists(localpath + '_bad-checksum_' + badsum)) 3035 self.assertTrue(os.path.exists(localpath)) 3036 3037 @skipIfNoNpm() 3038 @skipIfNoNetwork() 3039 def test_npmsw_premirrors(self): 3040 # Fetch once to get a tarball 3041 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 3042 ud = fetcher.ud[fetcher.urls[0]] 3043 fetcher.download() 3044 self.assertTrue(os.path.exists(ud.localpath)) 3045 # Setup the mirror 3046 mirrordir = os.path.join(self.tempdir, 'mirror') 3047 bb.utils.mkdirhier(mirrordir) 3048 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 3049 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 3050 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 3051 # Fetch again 3052 self.assertFalse(os.path.exists(ud.localpath)) 3053 swfile = self.create_shrinkwrap_file({ 3054 'packages': { 3055 'node_modules/array-flatten': { 3056 'version': '1.1.1', 3057 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3058 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3059 } 3060 } 3061 }) 3062 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3063 fetcher.download() 3064 self.assertTrue(os.path.exists(ud.localpath)) 3065 3066 @skipIfNoNpm() 3067 @skipIfNoNetwork() 3068 def test_npmsw_mirrors(self): 3069 # Fetch once to get a tarball 3070 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 3071 ud = fetcher.ud[fetcher.urls[0]] 3072 fetcher.download() 3073 self.assertTrue(os.path.exists(ud.localpath)) 3074 # Setup the mirror 3075 mirrordir = os.path.join(self.tempdir, 'mirror') 3076 bb.utils.mkdirhier(mirrordir) 3077 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 3078 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 3079 # Fetch again with invalid url 3080 self.assertFalse(os.path.exists(ud.localpath)) 3081 swfile = self.create_shrinkwrap_file({ 3082 'packages': { 3083 'node_modules/array-flatten': { 3084 'version': '1.1.1', 3085 'resolved': 'https://invalid', 3086 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3087 } 3088 } 3089 }) 3090 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3091 fetcher.download() 3092 self.assertTrue(os.path.exists(ud.localpath)) 3093 3094 @skipIfNoNetwork() 3095 def test_npmsw_bundled(self): 3096 swfile = self.create_shrinkwrap_file({ 3097 'packages': { 3098 'node_modules/array-flatten': { 3099 'version': '1.1.1', 3100 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 3101 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 3102 }, 3103 'node_modules/content-type': { 3104 'version': '1.0.4', 3105 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', 3106 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', 3107 'inBundle': True 3108 } 3109 } 3110 }) 3111 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 3112 fetcher.download() 3113 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 3114 self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 3115 3116class GitSharedTest(FetcherTest): 3117 def setUp(self): 3118 super(GitSharedTest, self).setUp() 3119 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https" 3120 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 3121 self.d.setVar("__BBSRCREV_SEEN", "1") 3122 3123 @skipIfNoNetwork() 3124 def test_shared_unpack(self): 3125 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3126 3127 fetcher.download() 3128 fetcher.unpack(self.unpackdir) 3129 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 3130 self.assertTrue(os.path.exists(alt)) 3131 3132 @skipIfNoNetwork() 3133 def test_noshared_unpack(self): 3134 self.d.setVar('BB_GIT_NOSHARED', '1') 3135 self.unpackdir += '_noshared' 3136 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3137 3138 fetcher.download() 3139 fetcher.unpack(self.unpackdir) 3140 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 3141 self.assertFalse(os.path.exists(alt)) 3142 3143 3144class FetchPremirroronlyLocalTest(FetcherTest): 3145 3146 def setUp(self): 3147 super(FetchPremirroronlyLocalTest, self).setUp() 3148 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3149 os.mkdir(self.mirrordir) 3150 self.reponame = "bitbake" 3151 self.gitdir = os.path.join(self.tempdir, "git", self.reponame) 3152 self.recipe_url = "git://git.fake.repo/bitbake;branch=master;protocol=https" 3153 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3154 self.d.setVar("BB_NO_NETWORK", "1") 3155 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3156 self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz" 3157 self.mirrorfile = os.path.join(self.mirrordir, self.mirrorname) 3158 self.testfilename = "bitbake-fetch.test" 3159 3160 def make_git_repo(self): 3161 recipeurl = "git:/git.fake.repo/bitbake" 3162 os.makedirs(self.gitdir) 3163 self.git_init(cwd=self.gitdir) 3164 for i in range(0): 3165 self.git_new_commit() 3166 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3167 3168 def git_new_commit(self): 3169 import random 3170 os.unlink(os.path.join(self.mirrordir, self.mirrorname)) 3171 branch = self.git("branch --show-current", self.gitdir).split() 3172 with open(os.path.join(self.gitdir, self.testfilename), "w") as testfile: 3173 testfile.write("File {} from branch {}; Useless random data {}".format(self.testfilename, branch, random.random())) 3174 self.git("add {}".format(self.testfilename), self.gitdir) 3175 self.git("commit -a -m \"This random commit {} in branch {}. I'm useless.\"".format(random.random(), branch), self.gitdir) 3176 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3177 return self.git("rev-parse HEAD", self.gitdir).strip() 3178 3179 def git_new_branch(self, name): 3180 self.git_new_commit() 3181 head = self.git("rev-parse HEAD", self.gitdir).strip() 3182 self.git("checkout -b {}".format(name), self.gitdir) 3183 newrev = self.git_new_commit() 3184 self.git("checkout {}".format(head), self.gitdir) 3185 return newrev 3186 3187 def test_mirror_multiple_fetches(self): 3188 self.make_git_repo() 3189 self.d.setVar("SRCREV", self.git_new_commit()) 3190 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3191 fetcher.download() 3192 fetcher.unpack(self.unpackdir) 3193 ## New commit in premirror. it's not in the download_dir 3194 self.d.setVar("SRCREV", self.git_new_commit()) 3195 fetcher2 = bb.fetch.Fetch([self.recipe_url], self.d) 3196 fetcher2.download() 3197 fetcher2.unpack(self.unpackdir) 3198 ## New commit in premirror. it's not in the download_dir 3199 self.d.setVar("SRCREV", self.git_new_commit()) 3200 fetcher3 = bb.fetch.Fetch([self.recipe_url], self.d) 3201 fetcher3.download() 3202 fetcher3.unpack(self.unpackdir) 3203 3204 3205 def test_mirror_commit_nonexistent(self): 3206 self.make_git_repo() 3207 self.d.setVar("SRCREV", "0"*40) 3208 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3209 with self.assertRaises(bb.fetch2.NetworkAccess): 3210 fetcher.download() 3211 3212 def test_mirror_commit_exists(self): 3213 self.make_git_repo() 3214 self.d.setVar("SRCREV", self.git_new_commit()) 3215 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3216 fetcher.download() 3217 fetcher.unpack(self.unpackdir) 3218 3219 def test_mirror_tarball_nonexistent(self): 3220 self.d.setVar("SRCREV", "0"*40) 3221 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3222 with self.assertRaises(bb.fetch2.NetworkAccess): 3223 fetcher.download() 3224 3225 def test_mirror_tarball_multiple_branches(self): 3226 """ 3227 test if PREMIRRORS can handle multiple name/branches correctly 3228 both branches have required revisions 3229 """ 3230 self.make_git_repo() 3231 branch1rev = self.git_new_branch("testbranch1") 3232 branch2rev = self.git_new_branch("testbranch2") 3233 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1,testbranch2;protocol=https;name=branch1,branch2" 3234 self.d.setVar("SRCREV_branch1", branch1rev) 3235 self.d.setVar("SRCREV_branch2", branch2rev) 3236 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3237 self.assertTrue(os.path.exists(self.mirrorfile), "Mirror file doesn't exist") 3238 fetcher.download() 3239 fetcher.unpack(os.path.join(self.tempdir, "unpacked")) 3240 unpacked = os.path.join(self.tempdir, "unpacked", "git", self.testfilename) 3241 self.assertTrue(os.path.exists(unpacked), "Repo has not been unpackaged properly!") 3242 with open(unpacked, 'r') as f: 3243 content = f.read() 3244 ## We expect to see testbranch1 in the file, not master, not testbranch2 3245 self.assertTrue(content.find("testbranch1") != -1, "Wrong branch has been checked out!") 3246 3247 def test_mirror_tarball_multiple_branches_nobranch(self): 3248 """ 3249 test if PREMIRRORS can handle multiple name/branches correctly 3250 Unbalanced name/branches raises ParameterError 3251 """ 3252 self.make_git_repo() 3253 branch1rev = self.git_new_branch("testbranch1") 3254 branch2rev = self.git_new_branch("testbranch2") 3255 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1;protocol=https;name=branch1,branch2" 3256 self.d.setVar("SRCREV_branch1", branch1rev) 3257 self.d.setVar("SRCREV_branch2", branch2rev) 3258 with self.assertRaises(bb.fetch2.ParameterError): 3259 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3260 3261 def test_mirror_tarball_multiple_branches_norev(self): 3262 """ 3263 test if PREMIRRORS can handle multiple name/branches correctly 3264 one of the branches specifies non existing SRCREV 3265 """ 3266 self.make_git_repo() 3267 branch1rev = self.git_new_branch("testbranch1") 3268 branch2rev = self.git_new_branch("testbranch2") 3269 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1,testbranch2;protocol=https;name=branch1,branch2" 3270 self.d.setVar("SRCREV_branch1", branch1rev) 3271 self.d.setVar("SRCREV_branch2", "0"*40) 3272 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3273 self.assertTrue(os.path.exists(self.mirrorfile), "Mirror file doesn't exist") 3274 with self.assertRaises(bb.fetch2.NetworkAccess): 3275 fetcher.download() 3276 3277 3278class FetchPremirroronlyNetworkTest(FetcherTest): 3279 3280 def setUp(self): 3281 super(FetchPremirroronlyNetworkTest, self).setUp() 3282 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3283 os.mkdir(self.mirrordir) 3284 self.reponame = "fstests" 3285 self.clonedir = os.path.join(self.tempdir, "git") 3286 self.gitdir = os.path.join(self.tempdir, "git", "{}.git".format(self.reponame)) 3287 self.recipe_url = "git://git.yoctoproject.org/fstests;protocol=https" 3288 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3289 self.d.setVar("BB_NO_NETWORK", "0") 3290 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3291 3292 def make_git_repo(self): 3293 import shutil 3294 self.mirrorname = "git2_git.yoctoproject.org.fstests.tar.gz" 3295 os.makedirs(self.clonedir) 3296 self.git("clone --bare --shallow-since=\"01.01.2013\" {}".format(self.recipe_url), self.clonedir) 3297 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3298 shutil.rmtree(self.clonedir) 3299 3300 @skipIfNoNetwork() 3301 def test_mirror_tarball_updated(self): 3302 self.make_git_repo() 3303 ## Upstream commit is in the mirror 3304 self.d.setVar("SRCREV", "49d65d53c2bf558ae6e9185af0f3af7b79d255ec") 3305 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3306 fetcher.download() 3307 3308 @skipIfNoNetwork() 3309 def test_mirror_tarball_outdated(self): 3310 self.make_git_repo() 3311 ## Upstream commit not in the mirror 3312 self.d.setVar("SRCREV", "15413486df1f5a5b5af699b6f3ba5f0984e52a9f") 3313 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3314 with self.assertRaises(bb.fetch2.NetworkAccess): 3315 fetcher.download() 3316 3317class FetchPremirroronlyMercurialTest(FetcherTest): 3318 """ Test for premirrors with mercurial repos 3319 the test covers also basic hg:// clone (see fetch_and_create_tarball 3320 """ 3321 def skipIfNoHg(): 3322 import shutil 3323 if not shutil.which('hg'): 3324 return unittest.skip('Mercurial not installed') 3325 return lambda f: f 3326 3327 def setUp(self): 3328 super(FetchPremirroronlyMercurialTest, self).setUp() 3329 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3330 os.mkdir(self.mirrordir) 3331 self.reponame = "libgnt" 3332 self.clonedir = os.path.join(self.tempdir, "hg") 3333 self.recipe_url = "hg://keep.imfreedom.org/libgnt;module=libgnt" 3334 self.d.setVar("SRCREV", "53e8b422faaf") 3335 self.mirrorname = "hg_libgnt_keep.imfreedom.org_.libgnt.tar.gz" 3336 3337 def fetch_and_create_tarball(self): 3338 """ 3339 Ask bitbake to download repo and prepare mirror tarball for us 3340 """ 3341 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") 3342 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3343 fetcher.download() 3344 mirrorfile = os.path.join(self.d.getVar("DL_DIR"), self.mirrorname) 3345 self.assertTrue(os.path.exists(mirrorfile), "Mirror tarball {} has not been created".format(mirrorfile)) 3346 ## moving tarball to mirror directory 3347 os.rename(mirrorfile, os.path.join(self.mirrordir, self.mirrorname)) 3348 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "0") 3349 3350 3351 @skipIfNoNetwork() 3352 @skipIfNoHg() 3353 def test_premirror_mercurial(self): 3354 self.fetch_and_create_tarball() 3355 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3356 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3357 self.d.setVar("BB_NO_NETWORK", "1") 3358 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3359 fetcher.download() 3360 3361class FetchPremirroronlyBrokenTarball(FetcherTest): 3362 3363 def setUp(self): 3364 super(FetchPremirroronlyBrokenTarball, self).setUp() 3365 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3366 os.mkdir(self.mirrordir) 3367 self.reponame = "bitbake" 3368 self.gitdir = os.path.join(self.tempdir, "git", self.reponame) 3369 self.recipe_url = "git://git.fake.repo/bitbake;protocol=https" 3370 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3371 self.d.setVar("BB_NO_NETWORK", "1") 3372 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3373 self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz" 3374 with open(os.path.join(self.mirrordir, self.mirrorname), 'w') as targz: 3375 targz.write("This is not tar.gz file!") 3376 3377 def test_mirror_broken_download(self): 3378 import sys 3379 self.d.setVar("SRCREV", "0"*40) 3380 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3381 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs() as logs: 3382 fetcher.download() 3383 output = "".join(logs.output) 3384 self.assertFalse(" not a git repository (or any parent up to mount point /)" in output) 3385 3386class GoModTest(FetcherTest): 3387 3388 @skipIfNoNetwork() 3389 def test_gomod_url(self): 3390 urls = ['gomod://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;' 3391 'sha256sum=9bb69aea32f1d59711701f9562d66432c9c0374205e5009d1d1a62f03fb4fdad'] 3392 3393 fetcher = bb.fetch2.Fetch(urls, self.d) 3394 ud = fetcher.ud[urls[0]] 3395 self.assertEqual(ud.url, 'https://proxy.golang.org/github.com/%21azure/azure-sdk-for-go/sdk/storage/azblob/%40v/v1.0.0.zip') 3396 3397 fetcher.download() 3398 fetcher.unpack(self.unpackdir) 3399 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3400 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.zip'))) 3401 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod'))) 3402 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')), 3403 '7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873') 3404 3405 @skipIfNoNetwork() 3406 def test_gomod_url_go_mod_only(self): 3407 urls = ['gomod://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;mod=1;' 3408 'sha256sum=7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873'] 3409 3410 fetcher = bb.fetch2.Fetch(urls, self.d) 3411 ud = fetcher.ud[urls[0]] 3412 self.assertEqual(ud.url, 'https://proxy.golang.org/github.com/%21azure/azure-sdk-for-go/sdk/storage/azblob/%40v/v1.0.0.mod') 3413 3414 fetcher.download() 3415 fetcher.unpack(self.unpackdir) 3416 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3417 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod'))) 3418 3419 @skipIfNoNetwork() 3420 def test_gomod_url_sha256sum_varflag(self): 3421 urls = ['gomod://gopkg.in/ini.v1;version=v1.67.0'] 3422 self.d.setVarFlag('SRC_URI', 'gopkg.in/ini.v1@v1.67.0.sha256sum', 'bd845dfc762a87a56e5a32a07770dc83e86976db7705d7f89c5dbafdc60b06c6') 3423 3424 fetcher = bb.fetch2.Fetch(urls, self.d) 3425 ud = fetcher.ud[urls[0]] 3426 self.assertEqual(ud.url, 'https://proxy.golang.org/gopkg.in/ini.v1/%40v/v1.67.0.zip') 3427 self.assertEqual(ud.parm['name'], 'gopkg.in/ini.v1@v1.67.0') 3428 3429 fetcher.download() 3430 fetcher.unpack(self.unpackdir) 3431 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3432 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip'))) 3433 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod'))) 3434 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')), 3435 '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1') 3436 3437 @skipIfNoNetwork() 3438 def test_gomod_url_no_go_mod_in_module(self): 3439 urls = ['gomod://gopkg.in/ini.v1;version=v1.67.0;' 3440 'sha256sum=bd845dfc762a87a56e5a32a07770dc83e86976db7705d7f89c5dbafdc60b06c6'] 3441 3442 fetcher = bb.fetch2.Fetch(urls, self.d) 3443 ud = fetcher.ud[urls[0]] 3444 self.assertEqual(ud.url, 'https://proxy.golang.org/gopkg.in/ini.v1/%40v/v1.67.0.zip') 3445 3446 fetcher.download() 3447 fetcher.unpack(self.unpackdir) 3448 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3449 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip'))) 3450 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod'))) 3451 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')), 3452 '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1') 3453 3454 @skipIfNoNetwork() 3455 def test_gomod_url_host_only(self): 3456 urls = ['gomod://go.opencensus.io;version=v0.24.0;' 3457 'sha256sum=203a767d7f8e7c1ebe5588220ad168d1e15b14ae70a636de7ca9a4a88a7e0d0c'] 3458 3459 fetcher = bb.fetch2.Fetch(urls, self.d) 3460 ud = fetcher.ud[urls[0]] 3461 self.assertEqual(ud.url, 'https://proxy.golang.org/go.opencensus.io/%40v/v0.24.0.zip') 3462 3463 fetcher.download() 3464 fetcher.unpack(self.unpackdir) 3465 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3466 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.zip'))) 3467 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod'))) 3468 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod')), 3469 '0dc9ccc660ad21cebaffd548f2cc6efa27891c68b4fbc1f8a3893b00f1acec96') 3470 3471class GoModGitTest(FetcherTest): 3472 3473 @skipIfNoNetwork() 3474 def test_gomodgit_url_repo(self): 3475 urls = ['gomodgit://golang.org/x/net;version=v0.9.0;' 3476 'repo=go.googlesource.com/net;' 3477 'srcrev=694cff8668bac64e0864b552bffc280cd27f21b1'] 3478 3479 fetcher = bb.fetch2.Fetch(urls, self.d) 3480 ud = fetcher.ud[urls[0]] 3481 self.assertEqual(ud.host, 'go.googlesource.com') 3482 self.assertEqual(ud.path, '/net') 3483 self.assertEqual(ud.names, ['golang.org/x/net@v0.9.0']) 3484 self.assertEqual(self.d.getVar('SRCREV_golang.org/x/net@v0.9.0'), '694cff8668bac64e0864b552bffc280cd27f21b1') 3485 3486 fetcher.download() 3487 self.assertTrue(os.path.exists(ud.localpath)) 3488 3489 fetcher.unpack(self.unpackdir) 3490 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') 3491 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'ed42bd05533fd84ae290a5d33ebd3695a0a2b06131beebd5450825bee8603aca'))) 3492 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3493 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.zip'))) 3494 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.mod'))) 3495 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.mod')), 3496 'c5d6851ede50ec1c001afb763040194b68961bf06997e2605e8bf06dcd2aeb2e') 3497 3498 @skipIfNoNetwork() 3499 def test_gomodgit_url_subdir(self): 3500 urls = ['gomodgit://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;' 3501 'repo=github.com/Azure/azure-sdk-for-go;subdir=sdk/storage/azblob;' 3502 'srcrev=ec928e0ed34db682b3f783d3739d1c538142e0c3'] 3503 3504 fetcher = bb.fetch2.Fetch(urls, self.d) 3505 ud = fetcher.ud[urls[0]] 3506 self.assertEqual(ud.host, 'github.com') 3507 self.assertEqual(ud.path, '/Azure/azure-sdk-for-go') 3508 self.assertEqual(ud.parm['subpath'], 'sdk/storage/azblob') 3509 self.assertEqual(ud.names, ['github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0']) 3510 self.assertEqual(self.d.getVar('SRCREV_github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0'), 'ec928e0ed34db682b3f783d3739d1c538142e0c3') 3511 3512 fetcher.download() 3513 self.assertTrue(os.path.exists(ud.localpath)) 3514 3515 fetcher.unpack(self.unpackdir) 3516 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') 3517 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'd31d6145676ed3066ce573a8198f326dea5be45a43b3d8f41ce7787fd71d66b3'))) 3518 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3519 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.zip'))) 3520 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod'))) 3521 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')), 3522 '7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873') 3523 3524 @skipIfNoNetwork() 3525 def test_gomodgit_url_srcrev_var(self): 3526 urls = ['gomodgit://gopkg.in/ini.v1;version=v1.67.0'] 3527 self.d.setVar('SRCREV_gopkg.in/ini.v1@v1.67.0', 'b2f570e5b5b844226bbefe6fb521d891f529a951') 3528 3529 fetcher = bb.fetch2.Fetch(urls, self.d) 3530 ud = fetcher.ud[urls[0]] 3531 self.assertEqual(ud.host, 'gopkg.in') 3532 self.assertEqual(ud.path, '/ini.v1') 3533 self.assertEqual(ud.names, ['gopkg.in/ini.v1@v1.67.0']) 3534 self.assertEqual(ud.parm['srcrev'], 'b2f570e5b5b844226bbefe6fb521d891f529a951') 3535 3536 fetcher.download() 3537 fetcher.unpack(self.unpackdir) 3538 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') 3539 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3'))) 3540 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3541 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip'))) 3542 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod'))) 3543 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')), 3544 '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1') 3545 3546 @skipIfNoNetwork() 3547 def test_gomodgit_url_no_go_mod_in_module(self): 3548 urls = ['gomodgit://gopkg.in/ini.v1;version=v1.67.0;' 3549 'srcrev=b2f570e5b5b844226bbefe6fb521d891f529a951'] 3550 3551 fetcher = bb.fetch2.Fetch(urls, self.d) 3552 ud = fetcher.ud[urls[0]] 3553 self.assertEqual(ud.host, 'gopkg.in') 3554 self.assertEqual(ud.path, '/ini.v1') 3555 self.assertEqual(ud.names, ['gopkg.in/ini.v1@v1.67.0']) 3556 self.assertEqual(self.d.getVar('SRCREV_gopkg.in/ini.v1@v1.67.0'), 'b2f570e5b5b844226bbefe6fb521d891f529a951') 3557 3558 fetcher.download() 3559 fetcher.unpack(self.unpackdir) 3560 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') 3561 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3'))) 3562 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3563 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip'))) 3564 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod'))) 3565 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')), 3566 '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1') 3567 3568 @skipIfNoNetwork() 3569 def test_gomodgit_url_host_only(self): 3570 urls = ['gomodgit://go.opencensus.io;version=v0.24.0;' 3571 'repo=github.com/census-instrumentation/opencensus-go;' 3572 'srcrev=b1a01ee95db0e690d91d7193d037447816fae4c5'] 3573 3574 fetcher = bb.fetch2.Fetch(urls, self.d) 3575 ud = fetcher.ud[urls[0]] 3576 self.assertEqual(ud.host, 'github.com') 3577 self.assertEqual(ud.path, '/census-instrumentation/opencensus-go') 3578 self.assertEqual(ud.names, ['go.opencensus.io@v0.24.0']) 3579 self.assertEqual(self.d.getVar('SRCREV_go.opencensus.io@v0.24.0'), 'b1a01ee95db0e690d91d7193d037447816fae4c5') 3580 3581 fetcher.download() 3582 fetcher.unpack(self.unpackdir) 3583 vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') 3584 self.assertTrue(os.path.exists(os.path.join(vcsdir, 'aae3ac7b2122ed3345654e6327855e9682f4a5350d63e93dbcfc51c4419df0e1'))) 3585 downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') 3586 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.zip'))) 3587 self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod'))) 3588 self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod')), 3589 '0dc9ccc660ad21cebaffd548f2cc6efa27891c68b4fbc1f8a3893b00f1acec96') 3590