1# 2# BitBake Tests for the Fetcher (fetch2/) 3# 4# Copyright (C) 2012 Richard Purdie 5# 6# SPDX-License-Identifier: GPL-2.0-only 7# 8 9import unittest 10import hashlib 11import tempfile 12import collections 13import os 14import signal 15import tarfile 16from bb.fetch2 import URI 17from bb.fetch2 import FetchMethod 18import bb 19from bb.tests.support.httpserver import HTTPService 20 21def skipIfNoNetwork(): 22 if os.environ.get("BB_SKIP_NETTESTS") == "yes": 23 return unittest.skip("network test") 24 return lambda f: f 25 26class TestTimeout(Exception): 27 pass 28 29class Timeout(): 30 31 def __init__(self, seconds): 32 self.seconds = seconds 33 34 def handle_timeout(self, signum, frame): 35 raise TestTimeout("Test failed: timeout reached") 36 37 def __enter__(self): 38 signal.signal(signal.SIGALRM, self.handle_timeout) 39 signal.alarm(self.seconds) 40 41 def __exit__(self, exc_type, exc_val, exc_tb): 42 signal.alarm(0) 43 44class URITest(unittest.TestCase): 45 test_uris = { 46 "http://www.google.com/index.html" : { 47 'uri': 'http://www.google.com/index.html', 48 'scheme': 'http', 49 'hostname': 'www.google.com', 50 'port': None, 51 'hostport': 'www.google.com', 52 'path': '/index.html', 53 'userinfo': '', 54 'username': '', 55 'password': '', 56 'params': {}, 57 'query': {}, 58 'relative': False 59 }, 60 "http://www.google.com/index.html;param1=value1" : { 61 'uri': 'http://www.google.com/index.html;param1=value1', 62 'scheme': 'http', 63 'hostname': 'www.google.com', 64 'port': None, 65 'hostport': 'www.google.com', 66 'path': '/index.html', 67 'userinfo': '', 68 'username': '', 69 'password': '', 70 'params': { 71 'param1': 'value1' 72 }, 73 'query': {}, 74 'relative': False 75 }, 76 "http://www.example.org/index.html?param1=value1" : { 77 'uri': 'http://www.example.org/index.html?param1=value1', 78 'scheme': 'http', 79 'hostname': 'www.example.org', 80 'port': None, 81 'hostport': 'www.example.org', 82 'path': '/index.html', 83 'userinfo': '', 84 'username': '', 85 'password': '', 86 'params': {}, 87 'query': { 88 'param1': 'value1' 89 }, 90 'relative': False 91 }, 92 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2" : { 93 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2', 94 'scheme': 'http', 95 'hostname': 'www.example.org', 96 'port': None, 97 'hostport': 'www.example.org', 98 'path': '/index.html', 99 'userinfo': '', 100 'username': '', 101 'password': '', 102 'params': { 103 'param2': 'value2' 104 }, 105 'query': { 106 'qparam1': 'qvalue1' 107 }, 108 'relative': False 109 }, 110 # Check that trailing semicolons are handled correctly 111 "http://www.example.org/index.html?qparam1=qvalue1;param2=value2;" : { 112 'uri': 'http://www.example.org/index.html?qparam1=qvalue1;param2=value2', 113 'scheme': 'http', 114 'hostname': 'www.example.org', 115 'port': None, 116 'hostport': 'www.example.org', 117 'path': '/index.html', 118 'userinfo': '', 119 'username': '', 120 'password': '', 121 'params': { 122 'param2': 'value2' 123 }, 124 'query': { 125 'qparam1': 'qvalue1' 126 }, 127 'relative': False 128 }, 129 "http://www.example.com:8080/index.html" : { 130 'uri': 'http://www.example.com:8080/index.html', 131 'scheme': 'http', 132 'hostname': 'www.example.com', 133 'port': 8080, 134 'hostport': 'www.example.com:8080', 135 'path': '/index.html', 136 'userinfo': '', 137 'username': '', 138 'password': '', 139 'params': {}, 140 'query': {}, 141 'relative': False 142 }, 143 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : { 144 'uri': 'cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg', 145 'scheme': 'cvs', 146 'hostname': 'cvs.handhelds.org', 147 'port': None, 148 'hostport': 'cvs.handhelds.org', 149 'path': '/cvs', 150 'userinfo': 'anoncvs', 151 'username': 'anoncvs', 152 'password': '', 153 'params': { 154 'module': 'familiar/dist/ipkg' 155 }, 156 'query': {}, 157 'relative': False 158 }, 159 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg": { 160 'uri': 'cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg', 161 'scheme': 'cvs', 162 'hostname': 'cvs.handhelds.org', 163 'port': None, 164 'hostport': 'cvs.handhelds.org', 165 'path': '/cvs', 166 'userinfo': 'anoncvs:anonymous', 167 'username': 'anoncvs', 168 'password': 'anonymous', 169 'params': collections.OrderedDict([ 170 ('tag', 'V0-99-81'), 171 ('module', 'familiar/dist/ipkg') 172 ]), 173 'query': {}, 174 'relative': False 175 }, 176 "file://example.diff": { # NOTE: Not RFC compliant! 177 'uri': 'file:example.diff', 178 'scheme': 'file', 179 'hostname': '', 180 'port': None, 181 'hostport': '', 182 'path': 'example.diff', 183 'userinfo': '', 184 'username': '', 185 'password': '', 186 'params': {}, 187 'query': {}, 188 'relative': True 189 }, 190 "file:example.diff": { # NOTE: RFC compliant version of the former 191 'uri': 'file:example.diff', 192 'scheme': 'file', 193 'hostname': '', 194 'port': None, 195 'hostport': '', 196 'path': 'example.diff', 197 'userinfo': '', 198 'userinfo': '', 199 'username': '', 200 'password': '', 201 'params': {}, 202 'query': {}, 203 'relative': True 204 }, 205 "file:///tmp/example.diff": { 206 'uri': 'file:///tmp/example.diff', 207 'scheme': 'file', 208 'hostname': '', 209 'port': None, 210 'hostport': '', 211 'path': '/tmp/example.diff', 212 'userinfo': '', 213 'userinfo': '', 214 'username': '', 215 'password': '', 216 'params': {}, 217 'query': {}, 218 'relative': False 219 }, 220 "git:///path/example.git": { 221 'uri': 'git:///path/example.git', 222 'scheme': 'git', 223 'hostname': '', 224 'port': None, 225 'hostport': '', 226 'path': '/path/example.git', 227 'userinfo': '', 228 'userinfo': '', 229 'username': '', 230 'password': '', 231 'params': {}, 232 'query': {}, 233 'relative': False 234 }, 235 "git:path/example.git": { 236 'uri': 'git:path/example.git', 237 'scheme': 'git', 238 'hostname': '', 239 'port': None, 240 'hostport': '', 241 'path': 'path/example.git', 242 'userinfo': '', 243 'userinfo': '', 244 'username': '', 245 'password': '', 246 'params': {}, 247 'query': {}, 248 'relative': True 249 }, 250 "git://example.net/path/example.git": { 251 'uri': 'git://example.net/path/example.git', 252 'scheme': 'git', 253 'hostname': 'example.net', 254 'port': None, 255 'hostport': 'example.net', 256 'path': '/path/example.git', 257 'userinfo': '', 258 'userinfo': '', 259 'username': '', 260 'password': '', 261 'params': {}, 262 'query': {}, 263 'relative': False 264 }, 265 "git://tfs-example.org:22/tfs/example%20path/example.git": { 266 'uri': 'git://tfs-example.org:22/tfs/example%20path/example.git', 267 'scheme': 'git', 268 'hostname': 'tfs-example.org', 269 'port': 22, 270 'hostport': 'tfs-example.org:22', 271 'path': '/tfs/example path/example.git', 272 'userinfo': '', 273 'userinfo': '', 274 'username': '', 275 'password': '', 276 'params': {}, 277 'query': {}, 278 'relative': False 279 }, 280 "http://somesite.net;someparam=1": { 281 'uri': 'http://somesite.net;someparam=1', 282 'scheme': 'http', 283 'hostname': 'somesite.net', 284 'port': None, 285 'hostport': 'somesite.net', 286 'path': '', 287 'userinfo': '', 288 'userinfo': '', 289 'username': '', 290 'password': '', 291 'params': {"someparam" : "1"}, 292 'query': {}, 293 'relative': False 294 }, 295 "file://somelocation;someparam=1": { 296 'uri': 'file:somelocation;someparam=1', 297 'scheme': 'file', 298 'hostname': '', 299 'port': None, 300 'hostport': '', 301 'path': 'somelocation', 302 'userinfo': '', 303 'userinfo': '', 304 'username': '', 305 'password': '', 306 'params': {"someparam" : "1"}, 307 'query': {}, 308 'relative': True 309 } 310 311 } 312 313 def test_uri(self): 314 for test_uri, ref in self.test_uris.items(): 315 uri = URI(test_uri) 316 317 self.assertEqual(str(uri), ref['uri']) 318 319 # expected attributes 320 self.assertEqual(uri.scheme, ref['scheme']) 321 322 self.assertEqual(uri.userinfo, ref['userinfo']) 323 self.assertEqual(uri.username, ref['username']) 324 self.assertEqual(uri.password, ref['password']) 325 326 self.assertEqual(uri.hostname, ref['hostname']) 327 self.assertEqual(uri.port, ref['port']) 328 self.assertEqual(uri.hostport, ref['hostport']) 329 330 self.assertEqual(uri.path, ref['path']) 331 self.assertEqual(uri.params, ref['params']) 332 333 self.assertEqual(uri.relative, ref['relative']) 334 335 def test_dict(self): 336 for test in self.test_uris.values(): 337 uri = URI() 338 339 self.assertEqual(uri.scheme, '') 340 self.assertEqual(uri.userinfo, '') 341 self.assertEqual(uri.username, '') 342 self.assertEqual(uri.password, '') 343 self.assertEqual(uri.hostname, '') 344 self.assertEqual(uri.port, None) 345 self.assertEqual(uri.path, '') 346 self.assertEqual(uri.params, {}) 347 348 349 uri.scheme = test['scheme'] 350 self.assertEqual(uri.scheme, test['scheme']) 351 352 uri.userinfo = test['userinfo'] 353 self.assertEqual(uri.userinfo, test['userinfo']) 354 self.assertEqual(uri.username, test['username']) 355 self.assertEqual(uri.password, test['password']) 356 357 # make sure changing the values doesn't do anything unexpected 358 uri.username = 'changeme' 359 self.assertEqual(uri.username, 'changeme') 360 self.assertEqual(uri.password, test['password']) 361 uri.password = 'insecure' 362 self.assertEqual(uri.username, 'changeme') 363 self.assertEqual(uri.password, 'insecure') 364 365 # reset back after our trickery 366 uri.userinfo = test['userinfo'] 367 self.assertEqual(uri.userinfo, test['userinfo']) 368 self.assertEqual(uri.username, test['username']) 369 self.assertEqual(uri.password, test['password']) 370 371 uri.hostname = test['hostname'] 372 self.assertEqual(uri.hostname, test['hostname']) 373 self.assertEqual(uri.hostport, test['hostname']) 374 375 uri.port = test['port'] 376 self.assertEqual(uri.port, test['port']) 377 self.assertEqual(uri.hostport, test['hostport']) 378 379 uri.path = test['path'] 380 self.assertEqual(uri.path, test['path']) 381 382 uri.params = test['params'] 383 self.assertEqual(uri.params, test['params']) 384 385 uri.query = test['query'] 386 self.assertEqual(uri.query, test['query']) 387 388 self.assertEqual(str(uri), test['uri']) 389 390 uri.params = {} 391 self.assertEqual(uri.params, {}) 392 self.assertEqual(str(uri), (str(uri).split(";"))[0]) 393 394class FetcherTest(unittest.TestCase): 395 396 def setUp(self): 397 self.origdir = os.getcwd() 398 self.d = bb.data.init() 399 self.tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-") 400 self.dldir = os.path.join(self.tempdir, "download") 401 os.mkdir(self.dldir) 402 self.d.setVar("DL_DIR", self.dldir) 403 self.unpackdir = os.path.join(self.tempdir, "unpacked") 404 os.mkdir(self.unpackdir) 405 persistdir = os.path.join(self.tempdir, "persistdata") 406 self.d.setVar("PERSISTENT_DIR", persistdir) 407 408 def tearDown(self): 409 os.chdir(self.origdir) 410 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes": 411 print("Not cleaning up %s. Please remove manually." % self.tempdir) 412 else: 413 bb.process.run('chmod u+rw -R %s' % self.tempdir) 414 bb.utils.prunedir(self.tempdir) 415 416 def git(self, cmd, cwd=None): 417 if isinstance(cmd, str): 418 cmd = 'git ' + cmd 419 else: 420 cmd = ['git'] + cmd 421 if cwd is None: 422 cwd = self.gitdir 423 return bb.process.run(cmd, cwd=cwd)[0] 424 425 def git_init(self, cwd=None): 426 self.git('init', cwd=cwd) 427 if not self.git(['config', 'user.email'], cwd=cwd): 428 self.git(['config', 'user.email', 'you@example.com'], cwd=cwd) 429 if not self.git(['config', 'user.name'], cwd=cwd): 430 self.git(['config', 'user.name', 'Your Name'], cwd=cwd) 431 432class MirrorUriTest(FetcherTest): 433 434 replaceuris = { 435 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/") 436 : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz", 437 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 438 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 439 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 440 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 441 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http") 442 : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 443 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake") 444 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890", 445 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache") 446 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 447 ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/") 448 : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 449 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3") 450 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", 451 ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz") 452 : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz", 453 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist") 454 : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", 455 ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/") 456 : "file:///somepath/downloads/subversion-1.7.1.tar.bz2", 457 ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 458 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 459 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 460 : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 461 ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http") 462 : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 463 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org") 464 : "http://somewhere2.org/somefile_1.2.3.tar.gz", 465 ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/") 466 : "http://somewhere2.org/somefile_1.2.3.tar.gz", 467 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http") 468 : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 469 ("git://user1@someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://user2@git.openembedded.org/bitbake;protocol=http") 470 : "git://user2@git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http", 471 ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=git;branch=master", "git://someserver.org/bitbake", "git://someotherserver.org/bitbake;protocol=https") 472 : "git://someotherserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=https;branch=master", 473 ("gitsm://git.qemu.org/git/seabios.git/;protocol=https;name=roms/seabios;subpath=roms/seabios;bareclone=1;nobranch=1;rev=1234567890123456789012345678901234567890", "gitsm://.*/.*", "http://petalinux.xilinx.com/sswreleases/rel-v${XILINX_VER_MAIN}/downloads") : "http://petalinux.xilinx.com/sswreleases/rel-v%24%7BXILINX_VER_MAIN%7D/downloads/git2_git.qemu.org.git.seabios.git..tar.gz", 474 ("https://somewhere.org/example/1.0.0/example;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/PATH") 475 : "file:///mirror/example/1.0.0/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", 476 ("https://somewhere.org/example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/some-example-1.0.0.tgz") 477 : "file:///mirror/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", 478 479 #Renaming files doesn't work 480 #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz" 481 #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz", 482 } 483 484 mirrorvar = "http://.*/.* file:///somepath/downloads/ " \ 485 "git://someserver.org/bitbake git://git.openembedded.org/bitbake " \ 486 "https://.*/.* file:///someotherpath/downloads/ " \ 487 "http://.*/.* file:///someotherpath/downloads/" 488 489 def test_urireplace(self): 490 self.d.setVar("FILESPATH", ".") 491 for k, v in self.replaceuris.items(): 492 ud = bb.fetch.FetchData(k[0], self.d) 493 ud.setup_localpath(self.d) 494 mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2])) 495 newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d) 496 self.assertEqual([v], newuris) 497 498 def test_urilist1(self): 499 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 500 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) 501 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 502 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 'file:///someotherpath/downloads/bitbake-1.0.tar.gz']) 503 504 def test_urilist2(self): 505 # Catch https:// -> files:// bug 506 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 507 mirrors = bb.fetch2.mirror_from_string(self.mirrorvar) 508 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 509 self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz']) 510 511 def test_mirror_of_mirror(self): 512 # Test if mirror of a mirror works 513 mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/" 514 mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/" 515 fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 516 mirrors = bb.fetch2.mirror_from_string(mirrorvar) 517 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 518 self.assertEqual(uris, ['file:///somepath/downloads/bitbake-1.0.tar.gz', 519 'file:///someotherpath/downloads/bitbake-1.0.tar.gz', 520 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz', 521 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz']) 522 523 recmirrorvar = "https://.*/[^/]* http://AAAA/A/A/A/ " \ 524 "https://.*/[^/]* https://BBBB/B/B/B/" 525 526 def test_recursive(self): 527 fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d) 528 mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar) 529 uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d) 530 self.assertEqual(uris, ['http://AAAA/A/A/A/bitbake/bitbake-1.0.tar.gz', 531 'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz', 532 'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz']) 533 534 535class GitDownloadDirectoryNamingTest(FetcherTest): 536 def setUp(self): 537 super(GitDownloadDirectoryNamingTest, self).setUp() 538 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master" 539 self.recipe_dir = "git.openembedded.org.bitbake" 540 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 541 self.mirror_dir = "github.com.openembedded.bitbake.git" 542 543 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 544 545 def setup_mirror_rewrite(self): 546 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 547 548 @skipIfNoNetwork() 549 def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self): 550 self.setup_mirror_rewrite() 551 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 552 553 fetcher.download() 554 555 dir = os.listdir(self.dldir + "/git2") 556 self.assertIn(self.recipe_dir, dir) 557 558 @skipIfNoNetwork() 559 def test_that_directory_exists_for_mirrored_url_and_recipe_url_when_mirroring_is_used(self): 560 self.setup_mirror_rewrite() 561 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 562 563 fetcher.download() 564 565 dir = os.listdir(self.dldir + "/git2") 566 self.assertIn(self.mirror_dir, dir) 567 self.assertIn(self.recipe_dir, dir) 568 569 @skipIfNoNetwork() 570 def test_that_recipe_directory_and_mirrored_directory_exists_when_mirroring_is_used_and_the_mirrored_directory_already_exists(self): 571 self.setup_mirror_rewrite() 572 fetcher = bb.fetch.Fetch([self.mirror_url], self.d) 573 fetcher.download() 574 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 575 576 fetcher.download() 577 578 dir = os.listdir(self.dldir + "/git2") 579 self.assertIn(self.mirror_dir, dir) 580 self.assertIn(self.recipe_dir, dir) 581 582 583class TarballNamingTest(FetcherTest): 584 def setUp(self): 585 super(TarballNamingTest, self).setUp() 586 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master" 587 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz" 588 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 589 self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz" 590 591 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1') 592 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 593 594 def setup_mirror_rewrite(self): 595 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 596 597 @skipIfNoNetwork() 598 def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self): 599 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 600 601 fetcher.download() 602 603 dir = os.listdir(self.dldir) 604 self.assertIn(self.recipe_tarball, dir) 605 606 @skipIfNoNetwork() 607 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self): 608 self.setup_mirror_rewrite() 609 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 610 611 fetcher.download() 612 613 dir = os.listdir(self.dldir) 614 self.assertIn(self.mirror_tarball, dir) 615 616 617class GitShallowTarballNamingTest(FetcherTest): 618 def setUp(self): 619 super(GitShallowTarballNamingTest, self).setUp() 620 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master" 621 self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz" 622 self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master" 623 self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz" 624 625 self.d.setVar('BB_GIT_SHALLOW', '1') 626 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 627 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 628 629 def setup_mirror_rewrite(self): 630 self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url) 631 632 @skipIfNoNetwork() 633 def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self): 634 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 635 636 fetcher.download() 637 638 dir = os.listdir(self.dldir) 639 self.assertIn(self.recipe_tarball, dir) 640 641 @skipIfNoNetwork() 642 def test_that_the_mirror_tarball_is_created_when_mirroring_is_used(self): 643 self.setup_mirror_rewrite() 644 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 645 646 fetcher.download() 647 648 dir = os.listdir(self.dldir) 649 self.assertIn(self.mirror_tarball, dir) 650 651 652class CleanTarballTest(FetcherTest): 653 def setUp(self): 654 super(CleanTarballTest, self).setUp() 655 self.recipe_url = "git://git.openembedded.org/bitbake" 656 self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz" 657 658 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1') 659 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 660 661 @skipIfNoNetwork() 662 def test_that_the_tarball_contents_does_not_leak_info(self): 663 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 664 665 fetcher.download() 666 667 fetcher.unpack(self.unpackdir) 668 mtime = bb.process.run('git log --all -1 --format=%ct', 669 cwd=os.path.join(self.unpackdir, 'git')) 670 self.assertEqual(len(mtime), 2) 671 mtime = int(mtime[0]) 672 673 archive = tarfile.open(os.path.join(self.dldir, self.recipe_tarball)) 674 self.assertNotEqual(len(archive.members), 0) 675 for member in archive.members: 676 self.assertEqual(member.uname, 'oe') 677 self.assertEqual(member.uid, 0) 678 self.assertEqual(member.gname, 'oe') 679 self.assertEqual(member.gid, 0) 680 self.assertEqual(member.mtime, mtime) 681 682 683class FetcherLocalTest(FetcherTest): 684 def setUp(self): 685 def touch(fn): 686 with open(fn, 'a'): 687 os.utime(fn, None) 688 689 super(FetcherLocalTest, self).setUp() 690 self.localsrcdir = os.path.join(self.tempdir, 'localsrc') 691 os.makedirs(self.localsrcdir) 692 touch(os.path.join(self.localsrcdir, 'a')) 693 touch(os.path.join(self.localsrcdir, 'b')) 694 os.makedirs(os.path.join(self.localsrcdir, 'dir')) 695 touch(os.path.join(self.localsrcdir, 'dir', 'c')) 696 touch(os.path.join(self.localsrcdir, 'dir', 'd')) 697 os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir')) 698 touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e')) 699 touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device')) 700 bb.process.run('tar cf archive.tar -C dir .', cwd=self.localsrcdir) 701 bb.process.run('tar czf archive.tar.gz -C dir .', cwd=self.localsrcdir) 702 bb.process.run('tar cjf archive.tar.bz2 -C dir .', cwd=self.localsrcdir) 703 self.d.setVar("FILESPATH", self.localsrcdir) 704 705 def fetchUnpack(self, uris): 706 fetcher = bb.fetch.Fetch(uris, self.d) 707 fetcher.download() 708 fetcher.unpack(self.unpackdir) 709 flst = [] 710 for root, dirs, files in os.walk(self.unpackdir): 711 for f in files: 712 flst.append(os.path.relpath(os.path.join(root, f), self.unpackdir)) 713 flst.sort() 714 return flst 715 716 def test_local_checksum_fails_no_file(self): 717 self.d.setVar("SRC_URI", "file://404") 718 with self.assertRaises(bb.BBHandledException): 719 bb.fetch.get_checksum_file_list(self.d) 720 721 def test_local(self): 722 tree = self.fetchUnpack(['file://a', 'file://dir/c']) 723 self.assertEqual(tree, ['a', 'dir/c']) 724 725 def test_local_backslash(self): 726 tree = self.fetchUnpack([r'file://backslash\x2dsystemd-unit.device']) 727 self.assertEqual(tree, [r'backslash\x2dsystemd-unit.device']) 728 729 def test_local_wildcard(self): 730 with self.assertRaises(bb.fetch2.ParameterError): 731 tree = self.fetchUnpack(['file://a', 'file://dir/*']) 732 733 def test_local_dir(self): 734 tree = self.fetchUnpack(['file://a', 'file://dir']) 735 self.assertEqual(tree, ['a', 'dir/c', 'dir/d', 'dir/subdir/e']) 736 737 def test_local_subdir(self): 738 tree = self.fetchUnpack(['file://dir/subdir']) 739 self.assertEqual(tree, ['dir/subdir/e']) 740 741 def test_local_subdir_file(self): 742 tree = self.fetchUnpack(['file://dir/subdir/e']) 743 self.assertEqual(tree, ['dir/subdir/e']) 744 745 def test_local_subdirparam(self): 746 tree = self.fetchUnpack(['file://a;subdir=bar', 'file://dir;subdir=foo/moo']) 747 self.assertEqual(tree, ['bar/a', 'foo/moo/dir/c', 'foo/moo/dir/d', 'foo/moo/dir/subdir/e']) 748 749 def test_local_deepsubdirparam(self): 750 tree = self.fetchUnpack(['file://dir/subdir/e;subdir=bar']) 751 self.assertEqual(tree, ['bar/dir/subdir/e']) 752 753 def test_local_absolutedir(self): 754 # Unpacking to an absolute path that is a subdirectory of the root 755 # should work 756 tree = self.fetchUnpack(['file://a;subdir=%s' % os.path.join(self.unpackdir, 'bar')]) 757 758 # Unpacking to an absolute path outside of the root should fail 759 with self.assertRaises(bb.fetch2.UnpackError): 760 self.fetchUnpack(['file://a;subdir=/bin/sh']) 761 762 def test_local_striplevel(self): 763 tree = self.fetchUnpack(['file://archive.tar;subdir=bar;striplevel=1']) 764 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 765 766 def test_local_striplevel_gzip(self): 767 tree = self.fetchUnpack(['file://archive.tar.gz;subdir=bar;striplevel=1']) 768 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 769 770 def test_local_striplevel_bzip2(self): 771 tree = self.fetchUnpack(['file://archive.tar.bz2;subdir=bar;striplevel=1']) 772 self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e']) 773 774 def dummyGitTest(self, suffix): 775 # Create dummy local Git repo 776 src_dir = tempfile.mkdtemp(dir=self.tempdir, 777 prefix='gitfetch_localusehead_') 778 self.gitdir = os.path.abspath(src_dir) 779 self.git_init() 780 self.git(['commit', '--allow-empty', '-m', 'Dummy commit']) 781 # Use other branch than master 782 self.git(['checkout', '-b', 'my-devel']) 783 self.git(['commit', '--allow-empty', '-m', 'Dummy commit 2']) 784 orig_rev = self.git(['rev-parse', 'HEAD']).strip() 785 786 # Fetch and check revision 787 self.d.setVar("SRCREV", "AUTOINC") 788 self.d.setVar("__BBSRCREV_SEEN", "1") 789 url = "git://" + self.gitdir + ";branch=master;protocol=file;" + suffix 790 fetcher = bb.fetch.Fetch([url], self.d) 791 fetcher.download() 792 fetcher.unpack(self.unpackdir) 793 unpack_rev = self.git(['rev-parse', 'HEAD'], 794 cwd=os.path.join(self.unpackdir, 'git')).strip() 795 self.assertEqual(orig_rev, unpack_rev) 796 797 def test_local_gitfetch_usehead(self): 798 self.dummyGitTest("usehead=1") 799 800 def test_local_gitfetch_usehead_withname(self): 801 self.dummyGitTest("usehead=1;name=newName") 802 803 def test_local_gitfetch_shared(self): 804 self.dummyGitTest("usehead=1;name=sharedName") 805 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 806 self.assertTrue(os.path.exists(alt)) 807 808 def test_local_gitfetch_noshared(self): 809 self.d.setVar('BB_GIT_NOSHARED', '1') 810 self.unpackdir += '_noshared' 811 self.dummyGitTest("usehead=1;name=noSharedName") 812 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 813 self.assertFalse(os.path.exists(alt)) 814 815class FetcherNoNetworkTest(FetcherTest): 816 def setUp(self): 817 super().setUp() 818 # all test cases are based on not having network 819 self.d.setVar("BB_NO_NETWORK", "1") 820 821 def test_missing(self): 822 string = "this is a test file\n".encode("utf-8") 823 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 824 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 825 826 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 827 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 828 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 829 with self.assertRaises(bb.fetch2.NetworkAccess): 830 fetcher.download() 831 832 def test_valid_missing_donestamp(self): 833 # create the file in the download directory with correct hash 834 string = "this is a test file\n".encode("utf-8") 835 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb") as f: 836 f.write(string) 837 838 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 839 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 840 841 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 842 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 843 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 844 fetcher.download() 845 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 846 847 def test_invalid_missing_donestamp(self): 848 # create an invalid file in the download directory with incorrect hash 849 string = "this is a test file\n".encode("utf-8") 850 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 851 pass 852 853 self.d.setVarFlag("SRC_URI", "md5sum", hashlib.md5(string).hexdigest()) 854 self.d.setVarFlag("SRC_URI", "sha256sum", hashlib.sha256(string).hexdigest()) 855 856 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 857 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 858 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/test-file.tar.gz"], self.d) 859 with self.assertRaises(bb.fetch2.NetworkAccess): 860 fetcher.download() 861 # the existing file should not exist or should have be moved to "bad-checksum" 862 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 863 864 def test_nochecksums_missing(self): 865 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 866 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 867 # ssh fetch does not support checksums 868 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 869 # attempts to download with missing donestamp 870 with self.assertRaises(bb.fetch2.NetworkAccess): 871 fetcher.download() 872 873 def test_nochecksums_missing_donestamp(self): 874 # create a file in the download directory 875 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 876 pass 877 878 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 879 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 880 # ssh fetch does not support checksums 881 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 882 # attempts to download with missing donestamp 883 with self.assertRaises(bb.fetch2.NetworkAccess): 884 fetcher.download() 885 886 def test_nochecksums_has_donestamp(self): 887 # create a file in the download directory with the donestamp 888 with open(os.path.join(self.dldir, "test-file.tar.gz"), "wb"): 889 pass 890 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"): 891 pass 892 893 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 894 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 895 # ssh fetch does not support checksums 896 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 897 # should not fetch 898 fetcher.download() 899 # both files should still exist 900 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 901 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 902 903 def test_nochecksums_missing_has_donestamp(self): 904 # create a file in the download directory with the donestamp 905 with open(os.path.join(self.dldir, "test-file.tar.gz.done"), "wb"): 906 pass 907 908 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 909 self.assertTrue(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 910 # ssh fetch does not support checksums 911 fetcher = bb.fetch.Fetch(["ssh://invalid@invalid.yoctoproject.org/test-file.tar.gz"], self.d) 912 with self.assertRaises(bb.fetch2.NetworkAccess): 913 fetcher.download() 914 # both files should still exist 915 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz"))) 916 self.assertFalse(os.path.exists(os.path.join(self.dldir, "test-file.tar.gz.done"))) 917 918class FetcherNetworkTest(FetcherTest): 919 @skipIfNoNetwork() 920 def test_fetch(self): 921 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) 922 fetcher.download() 923 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 924 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892) 925 self.d.setVar("BB_NO_NETWORK", "1") 926 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d) 927 fetcher.download() 928 fetcher.unpack(self.unpackdir) 929 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9) 930 self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.1/")), 9) 931 932 @skipIfNoNetwork() 933 def test_fetch_mirror(self): 934 self.d.setVar("MIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 935 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 936 fetcher.download() 937 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 938 939 @skipIfNoNetwork() 940 def test_fetch_mirror_of_mirror(self): 941 self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ http://invalid2.yoctoproject.org/.* https://downloads.yoctoproject.org/releases/bitbake") 942 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 943 fetcher.download() 944 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 945 946 @skipIfNoNetwork() 947 def test_fetch_file_mirror_of_mirror(self): 948 self.d.setVar("FILESPATH", ".") 949 self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ file:///some1where/.* file://some2where/ file://some2where/.* https://downloads.yoctoproject.org/releases/bitbake") 950 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 951 os.mkdir(self.dldir + "/some2where") 952 fetcher.download() 953 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 954 955 @skipIfNoNetwork() 956 def test_fetch_premirror(self): 957 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 958 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d) 959 fetcher.download() 960 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 961 962 @skipIfNoNetwork() 963 def test_fetch_specify_downloadfilename(self): 964 fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d) 965 fetcher.download() 966 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749) 967 968 @skipIfNoNetwork() 969 def test_fetch_premirror_specify_downloadfilename_regex_uri(self): 970 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake/") 971 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 972 fetcher.download() 973 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 974 975 @skipIfNoNetwork() 976 # BZ13039 977 def test_fetch_premirror_specify_downloadfilename_specific_uri(self): 978 self.d.setVar("PREMIRRORS", "http://invalid.yoctoproject.org/releases/bitbake https://downloads.yoctoproject.org/releases/bitbake") 979 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 980 fetcher.download() 981 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 982 983 @skipIfNoNetwork() 984 def test_fetch_premirror_use_downloadfilename_to_fetch(self): 985 # Ensure downloadfilename is used when fetching from premirror. 986 self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake") 987 fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d) 988 fetcher.download() 989 self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749) 990 991 @skipIfNoNetwork() 992 def gitfetcher(self, url1, url2): 993 def checkrevision(self, fetcher): 994 fetcher.unpack(self.unpackdir) 995 revision = self.git(['rev-parse', 'HEAD'], 996 cwd=os.path.join(self.unpackdir, 'git')).strip() 997 self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5") 998 999 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") 1000 self.d.setVar("SRCREV", "270a05b0b4ba0959fe0624d2a4885d7b70426da5") 1001 fetcher = bb.fetch.Fetch([url1], self.d) 1002 fetcher.download() 1003 checkrevision(self, fetcher) 1004 # Wipe out the dldir clone and the unpacked source, turn off the network and check mirror tarball works 1005 bb.utils.prunedir(self.dldir + "/git2/") 1006 bb.utils.prunedir(self.unpackdir) 1007 self.d.setVar("BB_NO_NETWORK", "1") 1008 fetcher = bb.fetch.Fetch([url2], self.d) 1009 fetcher.download() 1010 checkrevision(self, fetcher) 1011 1012 @skipIfNoNetwork() 1013 def test_gitfetch(self): 1014 url1 = url2 = "git://git.openembedded.org/bitbake;branch=master" 1015 self.gitfetcher(url1, url2) 1016 1017 @skipIfNoNetwork() 1018 def test_gitfetch_goodsrcrev(self): 1019 # SRCREV is set but matches rev= parameter 1020 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master" 1021 self.gitfetcher(url1, url2) 1022 1023 @skipIfNoNetwork() 1024 def test_gitfetch_badsrcrev(self): 1025 # SRCREV is set but does not match rev= parameter 1026 url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master" 1027 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) 1028 1029 @skipIfNoNetwork() 1030 def test_gitfetch_tagandrev(self): 1031 # SRCREV is set but does not match rev= parameter 1032 url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5" 1033 self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2) 1034 1035 @skipIfNoNetwork() 1036 def test_gitfetch_usehead(self): 1037 # Since self.gitfetcher() sets SRCREV we expect this to override 1038 # `usehead=1' and instead fetch the specified SRCREV. See 1039 # test_local_gitfetch_usehead() for a positive use of the usehead 1040 # feature. 1041 url = "git://git.openembedded.org/bitbake;usehead=1;branch=master" 1042 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url) 1043 1044 @skipIfNoNetwork() 1045 def test_gitfetch_usehead_withname(self): 1046 # Since self.gitfetcher() sets SRCREV we expect this to override 1047 # `usehead=1' and instead fetch the specified SRCREV. See 1048 # test_local_gitfetch_usehead() for a positive use of the usehead 1049 # feature. 1050 url = "git://git.openembedded.org/bitbake;usehead=1;name=newName;branch=master" 1051 self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url) 1052 1053 @skipIfNoNetwork() 1054 def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self): 1055 recipeurl = "git://git.openembedded.org/bitbake;branch=master" 1056 mirrorurl = "git://someserver.org/bitbake;branch=master" 1057 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake") 1058 self.gitfetcher(recipeurl, mirrorurl) 1059 1060 @skipIfNoNetwork() 1061 def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self): 1062 recipeurl = "git://someserver.org/bitbake;branch=master" 1063 self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake") 1064 self.gitfetcher(recipeurl, recipeurl) 1065 1066 @skipIfNoNetwork() 1067 def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self): 1068 realurl = "git://git.openembedded.org/bitbake" 1069 recipeurl = "git://someserver.org/bitbake" 1070 self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git") 1071 os.chdir(self.tempdir) 1072 self.git(['clone', realurl, self.sourcedir], cwd=self.tempdir) 1073 self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file" % (recipeurl, self.sourcedir)) 1074 self.gitfetcher(recipeurl, recipeurl) 1075 1076 @skipIfNoNetwork() 1077 def test_git_submodule(self): 1078 # URL with ssh submodules 1079 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7;branch=master" 1080 # Original URL (comment this if you have ssh access to git.yoctoproject.org) 1081 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee;branch=master" 1082 fetcher = bb.fetch.Fetch([url], self.d) 1083 fetcher.download() 1084 # Previous cwd has been deleted 1085 os.chdir(os.path.dirname(self.unpackdir)) 1086 fetcher.unpack(self.unpackdir) 1087 1088 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1089 self.assertTrue(os.path.exists(repo_path), msg='Unpacked repository missing') 1090 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake')), msg='bitbake submodule missing') 1091 self.assertFalse(os.path.exists(os.path.join(repo_path, 'na')), msg='uninitialized submodule present') 1092 1093 # Only when we're running the extended test with a submodule's submodule, can we check this. 1094 if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')): 1095 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing') 1096 1097 @skipIfNoNetwork() 1098 def test_git_submodule_dbus_broker(self): 1099 # The following external repositories have show failures in fetch and unpack operations 1100 # We want to avoid regressions! 1101 url = "gitsm://github.com/bus1/dbus-broker;protocol=https;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main" 1102 fetcher = bb.fetch.Fetch([url], self.d) 1103 fetcher.download() 1104 # Previous cwd has been deleted 1105 os.chdir(os.path.dirname(self.unpackdir)) 1106 fetcher.unpack(self.unpackdir) 1107 1108 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1109 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-dvar/config')), msg='Missing submodule config "subprojects/c-dvar"') 1110 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-list/config')), msg='Missing submodule config "subprojects/c-list"') 1111 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-rbtree/config')), msg='Missing submodule config "subprojects/c-rbtree"') 1112 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"') 1113 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"') 1114 1115 @skipIfNoNetwork() 1116 def test_git_submodule_CLI11(self): 1117 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main" 1118 fetcher = bb.fetch.Fetch([url], self.d) 1119 fetcher.download() 1120 # Previous cwd has been deleted 1121 os.chdir(os.path.dirname(self.unpackdir)) 1122 fetcher.unpack(self.unpackdir) 1123 1124 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1125 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"') 1126 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') 1127 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') 1128 1129 @skipIfNoNetwork() 1130 def test_git_submodule_update_CLI11(self): 1131 """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """ 1132 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main" 1133 fetcher = bb.fetch.Fetch([url], self.d) 1134 fetcher.download() 1135 1136 # CLI11 that pulls in a newer nlohmann-json 1137 url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main" 1138 fetcher = bb.fetch.Fetch([url], self.d) 1139 fetcher.download() 1140 # Previous cwd has been deleted 1141 os.chdir(os.path.dirname(self.unpackdir)) 1142 fetcher.unpack(self.unpackdir) 1143 1144 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1145 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/googletest/config')), msg='Missing submodule config "extern/googletest"') 1146 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') 1147 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') 1148 1149 @skipIfNoNetwork() 1150 def test_git_submodule_aktualizr(self): 1151 url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=https;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44" 1152 fetcher = bb.fetch.Fetch([url], self.d) 1153 fetcher.download() 1154 # Previous cwd has been deleted 1155 os.chdir(os.path.dirname(self.unpackdir)) 1156 fetcher.unpack(self.unpackdir) 1157 1158 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1159 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/config')), msg='Missing submodule config "partial/extern/isotp-c/config"') 1160 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/partial/extern/isotp-c/modules/deps/bitfield-c/config')), msg='Missing submodule config "partial/extern/isotp-c/modules/deps/bitfield-c/config"') 1161 self.assertTrue(os.path.exists(os.path.join(repo_path, 'partial/extern/isotp-c/deps/bitfield-c/.git')), msg="Submodule of submodule isotp-c did not unpack properly") 1162 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/tests/tuf-test-vectors/config')), msg='Missing submodule config "tests/tuf-test-vectors/config"') 1163 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"') 1164 self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"') 1165 1166 @skipIfNoNetwork() 1167 def test_git_submodule_iotedge(self): 1168 """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """ 1169 1170 # This repository also has submodules where the module (name), path and url do not align 1171 url = "gitsm://github.com/azure/iotedge.git;protocol=https;rev=d76e0316c6f324345d77c48a83ce836d09392699;branch=main" 1172 fetcher = bb.fetch.Fetch([url], self.d) 1173 fetcher.download() 1174 # Previous cwd has been deleted 1175 os.chdir(os.path.dirname(self.unpackdir)) 1176 fetcher.unpack(self.unpackdir) 1177 1178 repo_path = os.path.join(self.tempdir, 'unpacked', 'git') 1179 1180 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/README.md')), msg='Missing submodule checkout') 1181 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/ctest/README.md')), msg='Missing submodule checkout') 1182 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/testrunner/readme.md')), msg='Missing submodule checkout') 1183 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/readme.md')), msg='Missing submodule checkout') 1184 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') 1185 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/c-shared/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') 1186 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/README.md')), msg='Missing submodule checkout') 1187 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/README.md')), msg='Missing submodule checkout') 1188 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/ctest/README.md')), msg='Missing submodule checkout') 1189 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/testrunner/readme.md')), msg='Missing submodule checkout') 1190 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/readme.md')), msg='Missing submodule checkout') 1191 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') 1192 self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') 1193 1194 @skipIfNoNetwork() 1195 def test_git_submodule_reference_to_parent(self): 1196 self.recipe_url = "gitsm://github.com/gflags/gflags.git;protocol=https;branch=master" 1197 self.d.setVar("SRCREV", "14e1138441bbbb584160cb1c0a0426ec1bac35f1") 1198 with Timeout(60): 1199 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 1200 with self.assertRaises(bb.fetch2.FetchError): 1201 fetcher.download() 1202 1203class SVNTest(FetcherTest): 1204 def skipIfNoSvn(): 1205 import shutil 1206 if not shutil.which("svn"): 1207 return unittest.skip("svn not installed, tests being skipped") 1208 1209 if not shutil.which("svnadmin"): 1210 return unittest.skip("svnadmin not installed, tests being skipped") 1211 1212 return lambda f: f 1213 1214 @skipIfNoSvn() 1215 def setUp(self): 1216 """ Create a local repository """ 1217 1218 super(SVNTest, self).setUp() 1219 1220 # Create something we can fetch 1221 src_dir = tempfile.mkdtemp(dir=self.tempdir, 1222 prefix='svnfetch_srcdir_') 1223 src_dir = os.path.abspath(src_dir) 1224 bb.process.run("echo readme > README.md", cwd=src_dir) 1225 1226 # Store it in a local SVN repository 1227 repo_dir = tempfile.mkdtemp(dir=self.tempdir, 1228 prefix='svnfetch_localrepo_') 1229 repo_dir = os.path.abspath(repo_dir) 1230 bb.process.run("svnadmin create project", cwd=repo_dir) 1231 1232 self.repo_url = "file://%s/project" % repo_dir 1233 bb.process.run("svn import --non-interactive -m 'Initial import' %s %s/trunk" % (src_dir, self.repo_url), 1234 cwd=repo_dir) 1235 1236 bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir) 1237 # Github will emulate SVN. Use this to check if we're downloding... 1238 bb.process.run("svn propset svn:externals 'bitbake https://github.com/PhilipHazel/pcre2.git' .", 1239 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) 1240 bb.process.run("svn commit --non-interactive -m 'Add external'", 1241 cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) 1242 1243 self.src_dir = src_dir 1244 self.repo_dir = repo_dir 1245 1246 @skipIfNoSvn() 1247 def tearDown(self): 1248 os.chdir(self.origdir) 1249 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes": 1250 print("Not cleaning up %s. Please remove manually." % self.tempdir) 1251 else: 1252 bb.utils.prunedir(self.tempdir) 1253 1254 @skipIfNoSvn() 1255 @skipIfNoNetwork() 1256 def test_noexternal_svn(self): 1257 # Always match the rev count from setUp (currently rev 2) 1258 url = "svn://%s;module=trunk;protocol=file;rev=2" % self.repo_url.replace('file://', '') 1259 fetcher = bb.fetch.Fetch([url], self.d) 1260 fetcher.download() 1261 os.chdir(os.path.dirname(self.unpackdir)) 1262 fetcher.unpack(self.unpackdir) 1263 1264 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") 1265 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") 1266 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should NOT exist") 1267 self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should NOT exit") 1268 1269 @skipIfNoSvn() 1270 def test_external_svn(self): 1271 # Always match the rev count from setUp (currently rev 2) 1272 url = "svn://%s;module=trunk;protocol=file;externals=allowed;rev=2" % self.repo_url.replace('file://', '') 1273 fetcher = bb.fetch.Fetch([url], self.d) 1274 fetcher.download() 1275 os.chdir(os.path.dirname(self.unpackdir)) 1276 fetcher.unpack(self.unpackdir) 1277 1278 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") 1279 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") 1280 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should exist") 1281 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should exit") 1282 1283class TrustedNetworksTest(FetcherTest): 1284 def test_trusted_network(self): 1285 # Ensure trusted_network returns False when the host IS in the list. 1286 url = "git://Someserver.org/foo;rev=1;branch=master" 1287 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org") 1288 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1289 1290 def test_wild_trusted_network(self): 1291 # Ensure trusted_network returns true when the *.host IS in the list. 1292 url = "git://Someserver.org/foo;rev=1;branch=master" 1293 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1294 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1295 1296 def test_prefix_wild_trusted_network(self): 1297 # Ensure trusted_network returns true when the prefix matches *.host. 1298 url = "git://git.Someserver.org/foo;rev=1;branch=master" 1299 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1300 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1301 1302 def test_two_prefix_wild_trusted_network(self): 1303 # Ensure trusted_network returns true when the prefix matches *.host. 1304 url = "git://something.git.Someserver.org/foo;rev=1;branch=master" 1305 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org") 1306 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1307 1308 def test_port_trusted_network(self): 1309 # Ensure trusted_network returns True, even if the url specifies a port. 1310 url = "git://someserver.org:8080/foo;rev=1;branch=master" 1311 self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org") 1312 self.assertTrue(bb.fetch.trusted_network(self.d, url)) 1313 1314 def test_untrusted_network(self): 1315 # Ensure trusted_network returns False when the host is NOT in the list. 1316 url = "git://someserver.org/foo;rev=1;branch=master" 1317 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org") 1318 self.assertFalse(bb.fetch.trusted_network(self.d, url)) 1319 1320 def test_wild_untrusted_network(self): 1321 # Ensure trusted_network returns False when the host is NOT in the list. 1322 url = "git://*.someserver.org/foo;rev=1;branch=master" 1323 self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org") 1324 self.assertFalse(bb.fetch.trusted_network(self.d, url)) 1325 1326class URLHandle(unittest.TestCase): 1327 1328 datatable = { 1329 "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}), 1330 "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}), 1331 "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])), 1332 "git://git.openembedded.org/bitbake;branch=@foo" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo'}), 1333 "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}), 1334 r'git://s.o-me_ONE:!#$%^&*()-_={}[]\|:?,.<>~`@git.openembedded.org/bitbake;branch=main': ('git', 'git.openembedded.org', '/bitbake', 's.o-me_ONE', r'!#$%^&*()-_={}[]\|:?,.<>~`', {'branch': 'main'}), 1335 } 1336 # we require a pathname to encodeurl but users can still pass such urls to 1337 # decodeurl and we need to handle them 1338 decodedata = datatable.copy() 1339 decodedata.update({ 1340 "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}), 1341 "npmsw://some.registry.url;package=@pkg;version=latest": ('npmsw', 'some.registry.url', '/', '', '', {'package': '@pkg', 'version': 'latest'}), 1342 }) 1343 1344 def test_decodeurl(self): 1345 for k, v in self.decodedata.items(): 1346 result = bb.fetch.decodeurl(k) 1347 self.assertEqual(result, v) 1348 1349 def test_encodeurl(self): 1350 for k, v in self.datatable.items(): 1351 result = bb.fetch.encodeurl(v) 1352 self.assertEqual(result, k) 1353 1354class FetchLatestVersionTest(FetcherTest): 1355 1356 test_git_uris = { 1357 # version pattern "X.Y.Z" 1358 ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "") 1359 : "1.99.4", 1360 # version pattern "vX.Y" 1361 # mirror of git.infradead.org since network issues interfered with testing 1362 ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git;branch=master", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "") 1363 : "1.5.0", 1364 # version pattern "pkg_name-X.Y" 1365 # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing 1366 ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto;branch=master", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "") 1367 : "1.0", 1368 # version pattern "pkg_name-vX.Y.Z" 1369 ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git;branch=master", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "") 1370 : "1.4.0", 1371 # combination version pattern 1372 ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "") 1373 : "1.2.0", 1374 ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "") 1375 : "2014.01", 1376 # version pattern "yyyymmdd" 1377 ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "") 1378 : "20120614", 1379 # packages with a valid UPSTREAM_CHECK_GITTAGREGEX 1380 # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing 1381 ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap;branch=master", "ae0394e687f1a77e966cf72f895da91840dffb8f", r"(?P<pver>(\d+\.(\d\.?)*))") 1382 : "0.4.3", 1383 ("build-appliance-image", "git://git.yoctoproject.org/poky;branch=master", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P<pver>(([0-9][\.|_]?)+[0-9]))") 1384 : "11.0.0", 1385 ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))") 1386 : "1.3.59", 1387 ("remake", "git://github.com/rocky/remake.git;protocol=https;branch=master", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))") 1388 : "3.82+dbg0.9", 1389 } 1390 1391 test_wget_uris = { 1392 # 1393 # packages with versions inside directory name 1394 # 1395 # http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2 1396 ("util-linux", "/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2", "", "") 1397 : "2.24.2", 1398 # http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz 1399 ("enchant", "/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz", "", "") 1400 : "1.6.0", 1401 # http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz 1402 ("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz", "", "") 1403 : "2.8.12.1", 1404 # https://download.gnome.org/sources/libxml2/2.9/libxml2-2.9.14.tar.xz 1405 ("libxml2", "/software/libxml2/2.9/libxml2-2.9.14.tar.xz", "", "") 1406 : "2.10.3", 1407 # 1408 # packages with versions only in current directory 1409 # 1410 # https://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2 1411 ("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "") 1412 : "2.19", 1413 # https://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2 1414 ("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "") 1415 : "20120814", 1416 # 1417 # packages with "99" in the name of possible version 1418 # 1419 # http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz 1420 ("pulseaudio", "/software/pulseaudio/releases/pulseaudio-4.0.tar.xz", "", "") 1421 : "5.0", 1422 # http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2 1423 ("xserver-xorg", "/releases/individual/xserver/xorg-server-1.15.1.tar.bz2", "", "") 1424 : "1.15.1", 1425 # 1426 # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX 1427 # 1428 # http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2 1429 # https://github.com/apple/cups/releases 1430 ("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", "/apple/cups/releases", r"(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz") 1431 : "2.0.0", 1432 # http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz 1433 # http://ftp.debian.org/debian/pool/main/d/db5.3/ 1434 ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", r"(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz") 1435 : "5.3.10", 1436 # 1437 # packages where the tarball compression changed in the new version 1438 # 1439 # http://ftp.debian.org/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz 1440 ("minicom", "/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz", "", "") 1441 : "2.8", 1442 } 1443 1444 @skipIfNoNetwork() 1445 def test_git_latest_versionstring(self): 1446 for k, v in self.test_git_uris.items(): 1447 self.d.setVar("PN", k[0]) 1448 self.d.setVar("SRCREV", k[2]) 1449 self.d.setVar("UPSTREAM_CHECK_GITTAGREGEX", k[3]) 1450 ud = bb.fetch2.FetchData(k[1], self.d) 1451 pupver= ud.method.latest_versionstring(ud, self.d) 1452 verstring = pupver[0] 1453 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) 1454 r = bb.utils.vercmp_string(v, verstring) 1455 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) 1456 1457 def test_wget_latest_versionstring(self): 1458 testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata" 1459 server = HTTPService(testdata) 1460 server.start() 1461 port = server.port 1462 try: 1463 for k, v in self.test_wget_uris.items(): 1464 self.d.setVar("PN", k[0]) 1465 checkuri = "" 1466 if k[2]: 1467 checkuri = "http://localhost:%s/" % port + k[2] 1468 self.d.setVar("UPSTREAM_CHECK_URI", checkuri) 1469 self.d.setVar("UPSTREAM_CHECK_REGEX", k[3]) 1470 url = "http://localhost:%s/" % port + k[1] 1471 ud = bb.fetch2.FetchData(url, self.d) 1472 pupver = ud.method.latest_versionstring(ud, self.d) 1473 verstring = pupver[0] 1474 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) 1475 r = bb.utils.vercmp_string(v, verstring) 1476 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) 1477 finally: 1478 server.stop() 1479 1480 1481class FetchCheckStatusTest(FetcherTest): 1482 test_wget_uris = ["https://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz", 1483 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz", 1484 "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz", 1485 "https://yoctoproject.org/", 1486 "https://docs.yoctoproject.org", 1487 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz", 1488 "https://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz", 1489 "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz", 1490 # GitHub releases are hosted on Amazon S3, which doesn't support HEAD 1491 "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz" 1492 ] 1493 1494 @skipIfNoNetwork() 1495 def test_wget_checkstatus(self): 1496 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d) 1497 for u in self.test_wget_uris: 1498 with self.subTest(url=u): 1499 ud = fetch.ud[u] 1500 m = ud.method 1501 ret = m.checkstatus(fetch, ud, self.d) 1502 self.assertTrue(ret, msg="URI %s, can't check status" % (u)) 1503 1504 @skipIfNoNetwork() 1505 def test_wget_checkstatus_connection_cache(self): 1506 from bb.fetch2 import FetchConnectionCache 1507 1508 connection_cache = FetchConnectionCache() 1509 fetch = bb.fetch2.Fetch(self.test_wget_uris, self.d, 1510 connection_cache = connection_cache) 1511 1512 for u in self.test_wget_uris: 1513 with self.subTest(url=u): 1514 ud = fetch.ud[u] 1515 m = ud.method 1516 ret = m.checkstatus(fetch, ud, self.d) 1517 self.assertTrue(ret, msg="URI %s, can't check status" % (u)) 1518 1519 connection_cache.close_connections() 1520 1521 1522class GitMakeShallowTest(FetcherTest): 1523 def setUp(self): 1524 FetcherTest.setUp(self) 1525 self.gitdir = os.path.join(self.tempdir, 'gitshallow') 1526 bb.utils.mkdirhier(self.gitdir) 1527 self.git_init() 1528 1529 def assertRefs(self, expected_refs): 1530 actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines() 1531 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs).splitlines() 1532 self.assertEqual(sorted(full_expected), sorted(actual_refs)) 1533 1534 def assertRevCount(self, expected_count, args=None): 1535 if args is None: 1536 args = ['HEAD'] 1537 revs = self.git(['rev-list'] + args) 1538 actual_count = len(revs.splitlines()) 1539 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count)) 1540 1541 def make_shallow(self, args=None): 1542 if args is None: 1543 args = ['HEAD'] 1544 return bb.process.run([bb.fetch2.git.Git.make_shallow_path] + args, cwd=self.gitdir) 1545 1546 def add_empty_file(self, path, msg=None): 1547 if msg is None: 1548 msg = path 1549 open(os.path.join(self.gitdir, path), 'w').close() 1550 self.git(['add', path]) 1551 self.git(['commit', '-m', msg, path]) 1552 1553 def test_make_shallow_single_branch_no_merge(self): 1554 self.add_empty_file('a') 1555 self.add_empty_file('b') 1556 self.assertRevCount(2) 1557 self.make_shallow() 1558 self.assertRevCount(1) 1559 1560 def test_make_shallow_single_branch_one_merge(self): 1561 self.add_empty_file('a') 1562 self.add_empty_file('b') 1563 self.git('checkout -b a_branch') 1564 self.add_empty_file('c') 1565 self.git('checkout master') 1566 self.add_empty_file('d') 1567 self.git('merge --no-ff --no-edit a_branch') 1568 self.git('branch -d a_branch') 1569 self.add_empty_file('e') 1570 self.assertRevCount(6) 1571 self.make_shallow(['HEAD~2']) 1572 self.assertRevCount(5) 1573 1574 def test_make_shallow_at_merge(self): 1575 self.add_empty_file('a') 1576 self.git('checkout -b a_branch') 1577 self.add_empty_file('b') 1578 self.git('checkout master') 1579 self.git('merge --no-ff --no-edit a_branch') 1580 self.git('branch -d a_branch') 1581 self.assertRevCount(3) 1582 self.make_shallow() 1583 self.assertRevCount(1) 1584 1585 def test_make_shallow_annotated_tag(self): 1586 self.add_empty_file('a') 1587 self.add_empty_file('b') 1588 self.git('tag -a -m a_tag a_tag') 1589 self.assertRevCount(2) 1590 self.make_shallow(['a_tag']) 1591 self.assertRevCount(1) 1592 1593 def test_make_shallow_multi_ref(self): 1594 self.add_empty_file('a') 1595 self.add_empty_file('b') 1596 self.git('checkout -b a_branch') 1597 self.add_empty_file('c') 1598 self.git('checkout master') 1599 self.add_empty_file('d') 1600 self.git('checkout -b a_branch_2') 1601 self.add_empty_file('a_tag') 1602 self.git('tag a_tag') 1603 self.git('checkout master') 1604 self.git('branch -D a_branch_2') 1605 self.add_empty_file('e') 1606 self.assertRevCount(6, ['--all']) 1607 self.make_shallow() 1608 self.assertRevCount(5, ['--all']) 1609 1610 def test_make_shallow_multi_ref_trim(self): 1611 self.add_empty_file('a') 1612 self.git('checkout -b a_branch') 1613 self.add_empty_file('c') 1614 self.git('checkout master') 1615 self.assertRevCount(1) 1616 self.assertRevCount(2, ['--all']) 1617 self.assertRefs(['master', 'a_branch']) 1618 self.make_shallow(['-r', 'master', 'HEAD']) 1619 self.assertRevCount(1, ['--all']) 1620 self.assertRefs(['master']) 1621 1622 def test_make_shallow_noop(self): 1623 self.add_empty_file('a') 1624 self.assertRevCount(1) 1625 self.make_shallow() 1626 self.assertRevCount(1) 1627 1628 @skipIfNoNetwork() 1629 def test_make_shallow_bitbake(self): 1630 self.git('remote add origin https://github.com/openembedded/bitbake') 1631 self.git('fetch --tags origin') 1632 orig_revs = len(self.git('rev-list --all').splitlines()) 1633 self.make_shallow(['refs/tags/1.10.0']) 1634 self.assertRevCount(orig_revs - 1746, ['--all']) 1635 1636class GitShallowTest(FetcherTest): 1637 def setUp(self): 1638 FetcherTest.setUp(self) 1639 self.gitdir = os.path.join(self.tempdir, 'git') 1640 self.srcdir = os.path.join(self.tempdir, 'gitsource') 1641 1642 bb.utils.mkdirhier(self.srcdir) 1643 self.git_init(cwd=self.srcdir) 1644 self.d.setVar('WORKDIR', self.tempdir) 1645 self.d.setVar('S', self.gitdir) 1646 self.d.delVar('PREMIRRORS') 1647 self.d.delVar('MIRRORS') 1648 1649 uri = 'git://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 1650 self.d.setVar('SRC_URI', uri) 1651 self.d.setVar('SRCREV', '${AUTOREV}') 1652 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') 1653 1654 self.d.setVar('BB_GIT_SHALLOW', '1') 1655 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0') 1656 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 1657 self.d.setVar("__BBSRCREV_SEEN", "1") 1658 1659 def assertRefs(self, expected_refs, cwd=None): 1660 if cwd is None: 1661 cwd = self.gitdir 1662 actual_refs = self.git(['for-each-ref', '--format=%(refname)'], cwd=cwd).splitlines() 1663 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs, cwd=cwd).splitlines() 1664 self.assertEqual(sorted(set(full_expected)), sorted(set(actual_refs))) 1665 1666 def assertRevCount(self, expected_count, args=None, cwd=None): 1667 if args is None: 1668 args = ['HEAD'] 1669 if cwd is None: 1670 cwd = self.gitdir 1671 revs = self.git(['rev-list'] + args, cwd=cwd) 1672 actual_count = len(revs.splitlines()) 1673 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count)) 1674 1675 def add_empty_file(self, path, cwd=None, msg=None): 1676 if msg is None: 1677 msg = path 1678 if cwd is None: 1679 cwd = self.srcdir 1680 open(os.path.join(cwd, path), 'w').close() 1681 self.git(['add', path], cwd) 1682 self.git(['commit', '-m', msg, path], cwd) 1683 1684 def fetch(self, uri=None): 1685 if uri is None: 1686 uris = self.d.getVar('SRC_URI').split() 1687 uri = uris[0] 1688 d = self.d 1689 else: 1690 d = self.d.createCopy() 1691 d.setVar('SRC_URI', uri) 1692 uri = d.expand(uri) 1693 uris = [uri] 1694 1695 fetcher = bb.fetch2.Fetch(uris, d) 1696 fetcher.download() 1697 ud = fetcher.ud[uri] 1698 return fetcher, ud 1699 1700 def fetch_and_unpack(self, uri=None): 1701 fetcher, ud = self.fetch(uri) 1702 fetcher.unpack(self.d.getVar('WORKDIR')) 1703 assert os.path.exists(self.d.getVar('S')) 1704 return fetcher, ud 1705 1706 def fetch_shallow(self, uri=None, disabled=False, keepclone=False): 1707 """Fetch a uri, generating a shallow tarball, then unpack using it""" 1708 fetcher, ud = self.fetch_and_unpack(uri) 1709 assert os.path.exists(ud.clonedir), 'Git clone in DLDIR (%s) does not exist for uri %s' % (ud.clonedir, uri) 1710 1711 # Confirm that the unpacked repo is unshallow 1712 if not disabled: 1713 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0])) 1714 1715 # fetch and unpack, from the shallow tarball 1716 bb.utils.remove(self.gitdir, recurse=True) 1717 bb.process.run('chmod u+w -R "%s"' % ud.clonedir) 1718 bb.utils.remove(ud.clonedir, recurse=True) 1719 bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True) 1720 1721 # confirm that the unpacked repo is used when no git clone or git 1722 # mirror tarball is available 1723 fetcher, ud = self.fetch_and_unpack(uri) 1724 if not disabled: 1725 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is not shallow' % self.gitdir 1726 else: 1727 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is shallow' % self.gitdir 1728 return fetcher, ud 1729 1730 def test_shallow_disabled(self): 1731 self.add_empty_file('a') 1732 self.add_empty_file('b') 1733 self.assertRevCount(2, cwd=self.srcdir) 1734 1735 self.d.setVar('BB_GIT_SHALLOW', '0') 1736 self.fetch_shallow(disabled=True) 1737 self.assertRevCount(2) 1738 1739 def test_shallow_nobranch(self): 1740 self.add_empty_file('a') 1741 self.add_empty_file('b') 1742 self.assertRevCount(2, cwd=self.srcdir) 1743 1744 srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip() 1745 self.d.setVar('SRCREV', srcrev) 1746 uri = self.d.getVar('SRC_URI').split()[0] 1747 uri = '%s;nobranch=1;bare=1' % uri 1748 1749 self.fetch_shallow(uri) 1750 self.assertRevCount(1) 1751 1752 # shallow refs are used to ensure the srcrev sticks around when we 1753 # have no other branches referencing it 1754 self.assertRefs(['refs/shallow/default']) 1755 1756 def test_shallow_default_depth_1(self): 1757 # Create initial git repo 1758 self.add_empty_file('a') 1759 self.add_empty_file('b') 1760 self.assertRevCount(2, cwd=self.srcdir) 1761 1762 self.fetch_shallow() 1763 self.assertRevCount(1) 1764 1765 def test_shallow_depth_0_disables(self): 1766 self.add_empty_file('a') 1767 self.add_empty_file('b') 1768 self.assertRevCount(2, cwd=self.srcdir) 1769 1770 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 1771 self.fetch_shallow(disabled=True) 1772 self.assertRevCount(2) 1773 1774 def test_shallow_depth_default_override(self): 1775 self.add_empty_file('a') 1776 self.add_empty_file('b') 1777 self.assertRevCount(2, cwd=self.srcdir) 1778 1779 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '2') 1780 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '1') 1781 self.fetch_shallow() 1782 self.assertRevCount(1) 1783 1784 def test_shallow_depth_default_override_disable(self): 1785 self.add_empty_file('a') 1786 self.add_empty_file('b') 1787 self.add_empty_file('c') 1788 self.assertRevCount(3, cwd=self.srcdir) 1789 1790 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 1791 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '2') 1792 self.fetch_shallow() 1793 self.assertRevCount(2) 1794 1795 def test_current_shallow_out_of_date_clone(self): 1796 # Create initial git repo 1797 self.add_empty_file('a') 1798 self.add_empty_file('b') 1799 self.add_empty_file('c') 1800 self.assertRevCount(3, cwd=self.srcdir) 1801 1802 # Clone and generate mirror tarball 1803 fetcher, ud = self.fetch() 1804 1805 # Ensure we have a current mirror tarball, but an out of date clone 1806 self.git('update-ref refs/heads/master refs/heads/master~1', cwd=ud.clonedir) 1807 self.assertRevCount(2, cwd=ud.clonedir) 1808 1809 # Fetch and unpack, from the current tarball, not the out of date clone 1810 bb.utils.remove(self.gitdir, recurse=True) 1811 fetcher, ud = self.fetch() 1812 fetcher.unpack(self.d.getVar('WORKDIR')) 1813 self.assertRevCount(1) 1814 1815 def test_shallow_single_branch_no_merge(self): 1816 self.add_empty_file('a') 1817 self.add_empty_file('b') 1818 self.assertRevCount(2, cwd=self.srcdir) 1819 1820 self.fetch_shallow() 1821 self.assertRevCount(1) 1822 assert os.path.exists(os.path.join(self.gitdir, 'a')) 1823 assert os.path.exists(os.path.join(self.gitdir, 'b')) 1824 1825 def test_shallow_no_dangling(self): 1826 self.add_empty_file('a') 1827 self.add_empty_file('b') 1828 self.assertRevCount(2, cwd=self.srcdir) 1829 1830 self.fetch_shallow() 1831 self.assertRevCount(1) 1832 assert not self.git('fsck --dangling') 1833 1834 def test_shallow_srcrev_branch_truncation(self): 1835 self.add_empty_file('a') 1836 self.add_empty_file('b') 1837 b_commit = self.git('rev-parse HEAD', cwd=self.srcdir).rstrip() 1838 self.add_empty_file('c') 1839 self.assertRevCount(3, cwd=self.srcdir) 1840 1841 self.d.setVar('SRCREV', b_commit) 1842 self.fetch_shallow() 1843 1844 # The 'c' commit was removed entirely, and 'a' was removed from history 1845 self.assertRevCount(1, ['--all']) 1846 self.assertEqual(self.git('rev-parse HEAD').strip(), b_commit) 1847 assert os.path.exists(os.path.join(self.gitdir, 'a')) 1848 assert os.path.exists(os.path.join(self.gitdir, 'b')) 1849 assert not os.path.exists(os.path.join(self.gitdir, 'c')) 1850 1851 def test_shallow_ref_pruning(self): 1852 self.add_empty_file('a') 1853 self.add_empty_file('b') 1854 self.git('branch a_branch', cwd=self.srcdir) 1855 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir) 1856 self.assertRevCount(2, cwd=self.srcdir) 1857 1858 self.fetch_shallow() 1859 1860 self.assertRefs(['master', 'origin/master']) 1861 self.assertRevCount(1) 1862 1863 def test_shallow_submodules(self): 1864 self.add_empty_file('a') 1865 self.add_empty_file('b') 1866 1867 smdir = os.path.join(self.tempdir, 'gitsubmodule') 1868 bb.utils.mkdirhier(smdir) 1869 self.git_init(cwd=smdir) 1870 # Make this look like it was cloned from a remote... 1871 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) 1872 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) 1873 self.add_empty_file('asub', cwd=smdir) 1874 self.add_empty_file('bsub', cwd=smdir) 1875 1876 self.git('submodule init', cwd=self.srcdir) 1877 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir) 1878 self.git('submodule update', cwd=self.srcdir) 1879 self.git('commit -m submodule -a', cwd=self.srcdir) 1880 1881 uri = 'gitsm://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 1882 fetcher, ud = self.fetch_shallow(uri) 1883 1884 # Verify the main repository is shallow 1885 self.assertRevCount(1) 1886 1887 # Verify the gitsubmodule directory is present 1888 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule')) 1889 1890 # Verify the submodule is also shallow 1891 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule')) 1892 1893 def test_shallow_submodule_mirrors(self): 1894 self.add_empty_file('a') 1895 self.add_empty_file('b') 1896 1897 smdir = os.path.join(self.tempdir, 'gitsubmodule') 1898 bb.utils.mkdirhier(smdir) 1899 self.git_init(cwd=smdir) 1900 # Make this look like it was cloned from a remote... 1901 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) 1902 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) 1903 self.add_empty_file('asub', cwd=smdir) 1904 self.add_empty_file('bsub', cwd=smdir) 1905 1906 self.git('submodule init', cwd=self.srcdir) 1907 self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir) 1908 self.git('submodule update', cwd=self.srcdir) 1909 self.git('commit -m submodule -a', cwd=self.srcdir) 1910 1911 uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir 1912 1913 # Fetch once to generate the shallow tarball 1914 fetcher, ud = self.fetch(uri) 1915 1916 # Set up the mirror 1917 mirrordir = os.path.join(self.tempdir, 'mirror') 1918 bb.utils.rename(self.dldir, mirrordir) 1919 self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/' % mirrordir) 1920 1921 # Fetch from the mirror 1922 bb.utils.remove(self.dldir, recurse=True) 1923 bb.utils.remove(self.gitdir, recurse=True) 1924 self.fetch_and_unpack(uri) 1925 1926 # Verify the main repository is shallow 1927 self.assertRevCount(1) 1928 1929 # Verify the gitsubmodule directory is present 1930 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule')) 1931 1932 # Verify the submodule is also shallow 1933 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule')) 1934 1935 if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')): 1936 def test_shallow_annex(self): 1937 self.add_empty_file('a') 1938 self.add_empty_file('b') 1939 self.git('annex init', cwd=self.srcdir) 1940 open(os.path.join(self.srcdir, 'c'), 'w').close() 1941 self.git('annex add c', cwd=self.srcdir) 1942 self.git('commit --author "Foo Bar <foo@bar>" -m annex-c -a', cwd=self.srcdir) 1943 bb.process.run('chmod u+w -R %s' % self.srcdir) 1944 1945 uri = 'gitannex://%s;protocol=file;subdir=${S};branch=master' % self.srcdir 1946 fetcher, ud = self.fetch_shallow(uri) 1947 1948 self.assertRevCount(1) 1949 assert './.git/annex/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0] 1950 assert os.path.exists(os.path.join(self.gitdir, 'c')) 1951 1952 def test_shallow_multi_one_uri(self): 1953 # Create initial git repo 1954 self.add_empty_file('a') 1955 self.add_empty_file('b') 1956 self.git('checkout -b a_branch', cwd=self.srcdir) 1957 self.add_empty_file('c') 1958 self.add_empty_file('d') 1959 self.git('checkout master', cwd=self.srcdir) 1960 self.git('tag v0.0 a_branch', cwd=self.srcdir) 1961 self.add_empty_file('e') 1962 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 1963 self.add_empty_file('f') 1964 self.assertRevCount(7, cwd=self.srcdir) 1965 1966 uri = self.d.getVar('SRC_URI').split()[0] 1967 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri 1968 1969 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 1970 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 1971 self.d.setVar('SRCREV_master', '${AUTOREV}') 1972 self.d.setVar('SRCREV_a_branch', '${AUTOREV}') 1973 1974 self.fetch_shallow(uri) 1975 1976 self.assertRevCount(5) 1977 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 1978 1979 def test_shallow_multi_one_uri_depths(self): 1980 # Create initial git repo 1981 self.add_empty_file('a') 1982 self.add_empty_file('b') 1983 self.git('checkout -b a_branch', cwd=self.srcdir) 1984 self.add_empty_file('c') 1985 self.add_empty_file('d') 1986 self.git('checkout master', cwd=self.srcdir) 1987 self.add_empty_file('e') 1988 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 1989 self.add_empty_file('f') 1990 self.assertRevCount(7, cwd=self.srcdir) 1991 1992 uri = self.d.getVar('SRC_URI').split()[0] 1993 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri 1994 1995 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 1996 self.d.setVar('BB_GIT_SHALLOW_DEPTH_master', '3') 1997 self.d.setVar('BB_GIT_SHALLOW_DEPTH_a_branch', '1') 1998 self.d.setVar('SRCREV_master', '${AUTOREV}') 1999 self.d.setVar('SRCREV_a_branch', '${AUTOREV}') 2000 2001 self.fetch_shallow(uri) 2002 2003 self.assertRevCount(4, ['--all']) 2004 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 2005 2006 def test_shallow_clone_preferred_over_shallow(self): 2007 self.add_empty_file('a') 2008 self.add_empty_file('b') 2009 2010 # Fetch once to generate the shallow tarball 2011 fetcher, ud = self.fetch() 2012 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0])) 2013 2014 # Fetch and unpack with both the clonedir and shallow tarball available 2015 bb.utils.remove(self.gitdir, recurse=True) 2016 fetcher, ud = self.fetch_and_unpack() 2017 2018 # The unpacked tree should *not* be shallow 2019 self.assertRevCount(2) 2020 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')) 2021 2022 def test_shallow_mirrors(self): 2023 self.add_empty_file('a') 2024 self.add_empty_file('b') 2025 2026 # Fetch once to generate the shallow tarball 2027 fetcher, ud = self.fetch() 2028 mirrortarball = ud.mirrortarballs[0] 2029 assert os.path.exists(os.path.join(self.dldir, mirrortarball)) 2030 2031 # Set up the mirror 2032 mirrordir = os.path.join(self.tempdir, 'mirror') 2033 bb.utils.mkdirhier(mirrordir) 2034 self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/' % mirrordir) 2035 2036 bb.utils.rename(os.path.join(self.dldir, mirrortarball), 2037 os.path.join(mirrordir, mirrortarball)) 2038 2039 # Fetch from the mirror 2040 bb.utils.remove(self.dldir, recurse=True) 2041 bb.utils.remove(self.gitdir, recurse=True) 2042 self.fetch_and_unpack() 2043 self.assertRevCount(1) 2044 2045 def test_shallow_invalid_depth(self): 2046 self.add_empty_file('a') 2047 self.add_empty_file('b') 2048 2049 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '-12') 2050 with self.assertRaises(bb.fetch2.FetchError): 2051 self.fetch() 2052 2053 def test_shallow_invalid_depth_default(self): 2054 self.add_empty_file('a') 2055 self.add_empty_file('b') 2056 2057 self.d.setVar('BB_GIT_SHALLOW_DEPTH_default', '-12') 2058 with self.assertRaises(bb.fetch2.FetchError): 2059 self.fetch() 2060 2061 def test_shallow_extra_refs(self): 2062 self.add_empty_file('a') 2063 self.add_empty_file('b') 2064 self.git('branch a_branch', cwd=self.srcdir) 2065 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir) 2066 self.assertRevCount(2, cwd=self.srcdir) 2067 2068 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/a_branch') 2069 self.fetch_shallow() 2070 2071 self.assertRefs(['master', 'origin/master', 'origin/a_branch']) 2072 self.assertRevCount(1) 2073 2074 def test_shallow_extra_refs_wildcard(self): 2075 self.add_empty_file('a') 2076 self.add_empty_file('b') 2077 self.git('branch a_branch', cwd=self.srcdir) 2078 self.git('tag v1.0', cwd=self.srcdir) 2079 self.assertRefs(['master', 'a_branch', 'v1.0'], cwd=self.srcdir) 2080 self.assertRevCount(2, cwd=self.srcdir) 2081 2082 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*') 2083 self.fetch_shallow() 2084 2085 self.assertRefs(['master', 'origin/master', 'v1.0']) 2086 self.assertRevCount(1) 2087 2088 def test_shallow_missing_extra_refs(self): 2089 self.add_empty_file('a') 2090 self.add_empty_file('b') 2091 2092 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/heads/foo') 2093 with self.assertRaises(bb.fetch2.FetchError): 2094 self.fetch() 2095 2096 def test_shallow_missing_extra_refs_wildcard(self): 2097 self.add_empty_file('a') 2098 self.add_empty_file('b') 2099 2100 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*') 2101 self.fetch() 2102 2103 def test_shallow_remove_revs(self): 2104 # Create initial git repo 2105 self.add_empty_file('a') 2106 self.add_empty_file('b') 2107 self.git('checkout -b a_branch', cwd=self.srcdir) 2108 self.add_empty_file('c') 2109 self.add_empty_file('d') 2110 self.git('checkout master', cwd=self.srcdir) 2111 self.git('tag v0.0 a_branch', cwd=self.srcdir) 2112 self.add_empty_file('e') 2113 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 2114 self.git('branch -d a_branch', cwd=self.srcdir) 2115 self.add_empty_file('f') 2116 self.assertRevCount(7, cwd=self.srcdir) 2117 2118 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2119 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2120 2121 self.fetch_shallow() 2122 2123 self.assertRevCount(5) 2124 2125 def test_shallow_invalid_revs(self): 2126 self.add_empty_file('a') 2127 self.add_empty_file('b') 2128 2129 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2130 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2131 2132 with self.assertRaises(bb.fetch2.FetchError): 2133 self.fetch() 2134 2135 def test_shallow_fetch_missing_revs(self): 2136 self.add_empty_file('a') 2137 self.add_empty_file('b') 2138 fetcher, ud = self.fetch(self.d.getVar('SRC_URI')) 2139 self.git('tag v0.0 master', cwd=self.srcdir) 2140 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2141 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2142 self.fetch_shallow() 2143 2144 def test_shallow_fetch_missing_revs_fails(self): 2145 self.add_empty_file('a') 2146 self.add_empty_file('b') 2147 fetcher, ud = self.fetch(self.d.getVar('SRC_URI')) 2148 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2149 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') 2150 2151 with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm: 2152 self.fetch_shallow() 2153 self.assertIn("Unable to find revision v0.0 even from upstream", cm.output[0]) 2154 2155 @skipIfNoNetwork() 2156 def test_bitbake(self): 2157 self.git('remote add --mirror=fetch origin https://github.com/openembedded/bitbake', cwd=self.srcdir) 2158 self.git('config core.bare true', cwd=self.srcdir) 2159 self.git('fetch', cwd=self.srcdir) 2160 2161 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') 2162 # Note that the 1.10.0 tag is annotated, so this also tests 2163 # reference of an annotated vs unannotated tag 2164 self.d.setVar('BB_GIT_SHALLOW_REVS', '1.10.0') 2165 2166 self.fetch_shallow() 2167 2168 # Confirm that the history of 1.10.0 was removed 2169 orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines()) 2170 revs = len(self.git('rev-list master').splitlines()) 2171 self.assertNotEqual(orig_revs, revs) 2172 self.assertRefs(['master', 'origin/master']) 2173 self.assertRevCount(orig_revs - 1758) 2174 2175 def test_that_unpack_throws_an_error_when_the_git_clone_nor_shallow_tarball_exist(self): 2176 self.add_empty_file('a') 2177 fetcher, ud = self.fetch() 2178 bb.utils.remove(self.gitdir, recurse=True) 2179 bb.utils.remove(self.dldir, recurse=True) 2180 2181 with self.assertRaises(bb.fetch2.UnpackError) as context: 2182 fetcher.unpack(self.d.getVar('WORKDIR')) 2183 2184 self.assertIn("No up to date source found", context.exception.msg) 2185 self.assertIn("clone directory not available or not up to date", context.exception.msg) 2186 2187 @skipIfNoNetwork() 2188 def test_that_unpack_does_work_when_using_git_shallow_tarball_but_tarball_is_not_available(self): 2189 self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0') 2190 self.d.setVar('BB_GIT_SHALLOW', '1') 2191 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 2192 fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests;branch=master"], self.d) 2193 fetcher.download() 2194 2195 bb.utils.remove(self.dldir + "/*.tar.gz") 2196 fetcher.unpack(self.unpackdir) 2197 2198 dir = os.listdir(self.unpackdir + "/git/") 2199 self.assertIn("fstests.doap", dir) 2200 2201class GitLfsTest(FetcherTest): 2202 def skipIfNoGitLFS(): 2203 import shutil 2204 if not shutil.which('git-lfs'): 2205 return unittest.skip('git-lfs not installed') 2206 return lambda f: f 2207 2208 def setUp(self): 2209 FetcherTest.setUp(self) 2210 2211 self.gitdir = os.path.join(self.tempdir, 'git') 2212 self.srcdir = os.path.join(self.tempdir, 'gitsource') 2213 2214 self.d.setVar('WORKDIR', self.tempdir) 2215 self.d.setVar('S', self.gitdir) 2216 self.d.delVar('PREMIRRORS') 2217 self.d.delVar('MIRRORS') 2218 2219 self.d.setVar('SRCREV', '${AUTOREV}') 2220 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') 2221 self.d.setVar("__BBSRCREV_SEEN", "1") 2222 2223 bb.utils.mkdirhier(self.srcdir) 2224 self.git_init(cwd=self.srcdir) 2225 with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs: 2226 attrs.write('*.mp3 filter=lfs -text') 2227 self.git(['add', '.gitattributes'], cwd=self.srcdir) 2228 self.git(['commit', '-m', "attributes", '.gitattributes'], cwd=self.srcdir) 2229 2230 def fetch(self, uri=None, download=True): 2231 uris = self.d.getVar('SRC_URI').split() 2232 uri = uris[0] 2233 d = self.d 2234 2235 fetcher = bb.fetch2.Fetch(uris, d) 2236 if download: 2237 fetcher.download() 2238 ud = fetcher.ud[uri] 2239 return fetcher, ud 2240 2241 def get_real_git_lfs_file(self): 2242 self.d.setVar('PATH', os.environ.get('PATH')) 2243 fetcher, ud = self.fetch() 2244 fetcher.unpack(self.d.getVar('WORKDIR')) 2245 unpacked_lfs_file = os.path.join(self.d.getVar('WORKDIR'), 'git', "Cat_poster_1.jpg") 2246 return unpacked_lfs_file 2247 2248 @skipIfNoGitLFS() 2249 @skipIfNoNetwork() 2250 def test_real_git_lfs_repo_succeeds_without_lfs_param(self): 2251 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master") 2252 f = self.get_real_git_lfs_file() 2253 self.assertTrue(os.path.exists(f)) 2254 self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f)) 2255 2256 @skipIfNoGitLFS() 2257 @skipIfNoNetwork() 2258 def test_real_git_lfs_repo_succeeds(self): 2259 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=1") 2260 f = self.get_real_git_lfs_file() 2261 self.assertTrue(os.path.exists(f)) 2262 self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f)) 2263 2264 @skipIfNoGitLFS() 2265 @skipIfNoNetwork() 2266 def test_real_git_lfs_repo_succeeds(self): 2267 self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=0") 2268 f = self.get_real_git_lfs_file() 2269 # This is the actual non-smudged placeholder file on the repo if git-lfs does not run 2270 lfs_file = ( 2271 'version https://git-lfs.github.com/spec/v1\n' 2272 'oid sha256:34be66b1a39a1955b46a12588df9d5f6fc1da790e05cf01f3c7422f4bbbdc26b\n' 2273 'size 11423554\n' 2274 ) 2275 2276 with open(f) as fh: 2277 self.assertEqual(lfs_file, fh.read()) 2278 2279 def test_lfs_enabled(self): 2280 import shutil 2281 2282 uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir 2283 self.d.setVar('SRC_URI', uri) 2284 2285 # Careful: suppress initial attempt at downloading until 2286 # we know whether git-lfs is installed. 2287 fetcher, ud = self.fetch(uri=None, download=False) 2288 self.assertIsNotNone(ud.method._find_git_lfs) 2289 2290 # If git-lfs can be found, the unpack should be successful. Only 2291 # attempt this with the real live copy of git-lfs installed. 2292 if ud.method._find_git_lfs(self.d): 2293 fetcher.download() 2294 shutil.rmtree(self.gitdir, ignore_errors=True) 2295 fetcher.unpack(self.d.getVar('WORKDIR')) 2296 2297 old_find_git_lfs = ud.method._find_git_lfs 2298 try: 2299 # If git-lfs cannot be found, the unpack should throw an error 2300 with self.assertRaises(bb.fetch2.FetchError): 2301 fetcher.download() 2302 ud.method._find_git_lfs = lambda d: False 2303 shutil.rmtree(self.gitdir, ignore_errors=True) 2304 fetcher.unpack(self.d.getVar('WORKDIR')) 2305 finally: 2306 ud.method._find_git_lfs = old_find_git_lfs 2307 2308 def test_lfs_disabled(self): 2309 import shutil 2310 2311 uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir 2312 self.d.setVar('SRC_URI', uri) 2313 2314 # In contrast to test_lfs_enabled(), allow the implicit download 2315 # done by self.fetch() to occur here. The point of this test case 2316 # is to verify that the fetcher can survive even if the source 2317 # repository has Git LFS usage configured. 2318 fetcher, ud = self.fetch() 2319 self.assertIsNotNone(ud.method._find_git_lfs) 2320 2321 old_find_git_lfs = ud.method._find_git_lfs 2322 try: 2323 # If git-lfs can be found, the unpack should be successful. A 2324 # live copy of git-lfs is not required for this case, so 2325 # unconditionally forge its presence. 2326 ud.method._find_git_lfs = lambda d: True 2327 shutil.rmtree(self.gitdir, ignore_errors=True) 2328 fetcher.unpack(self.d.getVar('WORKDIR')) 2329 # If git-lfs cannot be found, the unpack should be successful 2330 2331 ud.method._find_git_lfs = lambda d: False 2332 shutil.rmtree(self.gitdir, ignore_errors=True) 2333 fetcher.unpack(self.d.getVar('WORKDIR')) 2334 finally: 2335 ud.method._find_git_lfs = old_find_git_lfs 2336 2337class GitURLWithSpacesTest(FetcherTest): 2338 test_git_urls = { 2339 "git://tfs-example.org:22/tfs/example%20path/example.git;branch=master" : { 2340 'url': 'git://tfs-example.org:22/tfs/example%20path/example.git;branch=master', 2341 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example.git', 2342 'path': '/tfs/example path/example.git' 2343 }, 2344 "git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master" : { 2345 'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master', 2346 'gitsrcname': 'tfs-example.org.22.tfs.example_path.example_repo.git', 2347 'path': '/tfs/example path/example repo.git' 2348 } 2349 } 2350 2351 def test_urls(self): 2352 2353 # Set fake SRCREV to stop git fetcher from trying to contact non-existent git repo 2354 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 2355 2356 for test_git_url, ref in self.test_git_urls.items(): 2357 2358 fetcher = bb.fetch.Fetch([test_git_url], self.d) 2359 ud = fetcher.ud[fetcher.urls[0]] 2360 2361 self.assertEqual(ud.url, ref['url']) 2362 self.assertEqual(ud.path, ref['path']) 2363 self.assertEqual(ud.localfile, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2364 self.assertEqual(ud.localpath, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2365 self.assertEqual(ud.lockfile, os.path.join(self.dldir, "git2", ref['gitsrcname'] + '.lock')) 2366 self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname'])) 2367 self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz')) 2368 2369class CrateTest(FetcherTest): 2370 @skipIfNoNetwork() 2371 def test_crate_url(self): 2372 2373 uri = "crate://crates.io/glob/0.2.11" 2374 self.d.setVar('SRC_URI', uri) 2375 2376 uris = self.d.getVar('SRC_URI').split() 2377 d = self.d 2378 2379 fetcher = bb.fetch2.Fetch(uris, self.d) 2380 ud = fetcher.ud[fetcher.urls[0]] 2381 2382 self.assertIn("name", ud.parm) 2383 self.assertEqual(ud.parm["name"], "glob") 2384 self.assertIn("downloadfilename", ud.parm) 2385 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") 2386 2387 fetcher.download() 2388 fetcher.unpack(self.tempdir) 2389 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2390 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done']) 2391 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) 2392 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) 2393 2394 @skipIfNoNetwork() 2395 def test_crate_url_params(self): 2396 2397 uri = "crate://crates.io/aho-corasick/0.7.20;name=aho-corasick-renamed" 2398 self.d.setVar('SRC_URI', uri) 2399 2400 uris = self.d.getVar('SRC_URI').split() 2401 d = self.d 2402 2403 fetcher = bb.fetch2.Fetch(uris, self.d) 2404 ud = fetcher.ud[fetcher.urls[0]] 2405 2406 self.assertIn("name", ud.parm) 2407 self.assertEqual(ud.parm["name"], "aho-corasick-renamed") 2408 self.assertIn("downloadfilename", ud.parm) 2409 self.assertEqual(ud.parm["downloadfilename"], "aho-corasick-0.7.20.crate") 2410 2411 fetcher.download() 2412 fetcher.unpack(self.tempdir) 2413 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2414 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['aho-corasick-0.7.20.crate', 'aho-corasick-0.7.20.crate.done']) 2415 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/.cargo-checksum.json")) 2416 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/src/lib.rs")) 2417 2418 @skipIfNoNetwork() 2419 def test_crate_url_multi(self): 2420 2421 uri = "crate://crates.io/glob/0.2.11 crate://crates.io/time/0.1.35" 2422 self.d.setVar('SRC_URI', uri) 2423 2424 uris = self.d.getVar('SRC_URI').split() 2425 d = self.d 2426 2427 fetcher = bb.fetch2.Fetch(uris, self.d) 2428 ud = fetcher.ud[fetcher.urls[0]] 2429 2430 self.assertIn("name", ud.parm) 2431 self.assertEqual(ud.parm["name"], "glob") 2432 self.assertIn("downloadfilename", ud.parm) 2433 self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate") 2434 2435 ud = fetcher.ud[fetcher.urls[1]] 2436 self.assertIn("name", ud.parm) 2437 self.assertEqual(ud.parm["name"], "time") 2438 self.assertIn("downloadfilename", ud.parm) 2439 self.assertEqual(ud.parm["downloadfilename"], "time-0.1.35.crate") 2440 2441 fetcher.download() 2442 fetcher.unpack(self.tempdir) 2443 self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked']) 2444 self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done', 'time-0.1.35.crate', 'time-0.1.35.crate.done']) 2445 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json")) 2446 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs")) 2447 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/.cargo-checksum.json")) 2448 self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/src/lib.rs")) 2449 2450 @skipIfNoNetwork() 2451 def test_crate_incorrect_cksum(self): 2452 uri = "crate://crates.io/aho-corasick/0.7.20" 2453 self.d.setVar('SRC_URI', uri) 2454 self.d.setVarFlag("SRC_URI", "aho-corasick.sha256sum", hashlib.sha256("Invalid".encode("utf-8")).hexdigest()) 2455 2456 uris = self.d.getVar('SRC_URI').split() 2457 2458 fetcher = bb.fetch2.Fetch(uris, self.d) 2459 with self.assertRaisesRegexp(bb.fetch2.FetchError, "Fetcher failure for URL"): 2460 fetcher.download() 2461 2462class NPMTest(FetcherTest): 2463 def skipIfNoNpm(): 2464 import shutil 2465 if not shutil.which('npm'): 2466 return unittest.skip('npm not installed') 2467 return lambda f: f 2468 2469 @skipIfNoNpm() 2470 @skipIfNoNetwork() 2471 def test_npm(self): 2472 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2473 fetcher = bb.fetch.Fetch([url], self.d) 2474 ud = fetcher.ud[fetcher.urls[0]] 2475 fetcher.download() 2476 self.assertTrue(os.path.exists(ud.localpath)) 2477 self.assertTrue(os.path.exists(ud.localpath + '.done')) 2478 self.assertTrue(os.path.exists(ud.resolvefile)) 2479 fetcher.unpack(self.unpackdir) 2480 unpackdir = os.path.join(self.unpackdir, 'npm') 2481 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2482 2483 @skipIfNoNpm() 2484 @skipIfNoNetwork() 2485 def test_npm_bad_checksum(self): 2486 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2487 # Fetch once to get a tarball 2488 fetcher = bb.fetch.Fetch([url], self.d) 2489 ud = fetcher.ud[fetcher.urls[0]] 2490 fetcher.download() 2491 self.assertTrue(os.path.exists(ud.localpath)) 2492 # Modify the tarball 2493 bad = b'bad checksum' 2494 with open(ud.localpath, 'wb') as f: 2495 f.write(bad) 2496 # Verify that the tarball is fetched again 2497 fetcher.download() 2498 badsum = hashlib.sha512(bad).hexdigest() 2499 self.assertTrue(os.path.exists(ud.localpath + '_bad-checksum_' + badsum)) 2500 self.assertTrue(os.path.exists(ud.localpath)) 2501 2502 @skipIfNoNpm() 2503 @skipIfNoNetwork() 2504 def test_npm_premirrors(self): 2505 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2506 # Fetch once to get a tarball 2507 fetcher = bb.fetch.Fetch([url], self.d) 2508 ud = fetcher.ud[fetcher.urls[0]] 2509 fetcher.download() 2510 self.assertTrue(os.path.exists(ud.localpath)) 2511 2512 # Setup the mirror by renaming the download directory 2513 mirrordir = os.path.join(self.tempdir, 'mirror') 2514 bb.utils.rename(self.dldir, mirrordir) 2515 os.mkdir(self.dldir) 2516 2517 # Configure the premirror to be used 2518 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/npm2' % mirrordir) 2519 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2520 2521 # Fetch again 2522 self.assertFalse(os.path.exists(ud.localpath)) 2523 # The npm fetcher doesn't handle that the .resolved file disappears 2524 # while the fetcher object exists, which it does when we rename the 2525 # download directory to "mirror" above. Thus we need a new fetcher to go 2526 # with the now empty download directory. 2527 fetcher = bb.fetch.Fetch([url], self.d) 2528 ud = fetcher.ud[fetcher.urls[0]] 2529 fetcher.download() 2530 self.assertTrue(os.path.exists(ud.localpath)) 2531 2532 @skipIfNoNpm() 2533 @skipIfNoNetwork() 2534 def test_npm_premirrors_with_specified_filename(self): 2535 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2536 # Fetch once to get a tarball 2537 fetcher = bb.fetch.Fetch([url], self.d) 2538 ud = fetcher.ud[fetcher.urls[0]] 2539 fetcher.download() 2540 self.assertTrue(os.path.exists(ud.localpath)) 2541 # Setup the mirror 2542 mirrordir = os.path.join(self.tempdir, 'mirror') 2543 bb.utils.mkdirhier(mirrordir) 2544 mirrorfilename = os.path.join(mirrordir, os.path.basename(ud.localpath)) 2545 os.replace(ud.localpath, mirrorfilename) 2546 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s' % mirrorfilename) 2547 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2548 # Fetch again 2549 self.assertFalse(os.path.exists(ud.localpath)) 2550 fetcher.download() 2551 self.assertTrue(os.path.exists(ud.localpath)) 2552 2553 @skipIfNoNpm() 2554 @skipIfNoNetwork() 2555 def test_npm_mirrors(self): 2556 # Fetch once to get a tarball 2557 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2558 fetcher = bb.fetch.Fetch([url], self.d) 2559 ud = fetcher.ud[fetcher.urls[0]] 2560 fetcher.download() 2561 self.assertTrue(os.path.exists(ud.localpath)) 2562 # Setup the mirror 2563 mirrordir = os.path.join(self.tempdir, 'mirror') 2564 bb.utils.mkdirhier(mirrordir) 2565 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 2566 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 2567 # Update the resolved url to an invalid url 2568 with open(ud.resolvefile, 'r') as f: 2569 url = f.read() 2570 uri = URI(url) 2571 uri.path = '/invalid' 2572 with open(ud.resolvefile, 'w') as f: 2573 f.write(str(uri)) 2574 # Fetch again 2575 self.assertFalse(os.path.exists(ud.localpath)) 2576 fetcher.download() 2577 self.assertTrue(os.path.exists(ud.localpath)) 2578 2579 @skipIfNoNpm() 2580 @skipIfNoNetwork() 2581 def test_npm_destsuffix_downloadfilename(self): 2582 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz' 2583 fetcher = bb.fetch.Fetch([url], self.d) 2584 fetcher.download() 2585 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'foo-bar.tgz'))) 2586 fetcher.unpack(self.unpackdir) 2587 unpackdir = os.path.join(self.unpackdir, 'foo', 'bar') 2588 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2589 2590 def test_npm_no_network_no_tarball(self): 2591 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2592 self.d.setVar('BB_NO_NETWORK', '1') 2593 fetcher = bb.fetch.Fetch([url], self.d) 2594 with self.assertRaises(bb.fetch2.NetworkAccess): 2595 fetcher.download() 2596 2597 @skipIfNoNpm() 2598 @skipIfNoNetwork() 2599 def test_npm_no_network_with_tarball(self): 2600 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2601 # Fetch once to get a tarball 2602 fetcher = bb.fetch.Fetch([url], self.d) 2603 fetcher.download() 2604 # Disable network access 2605 self.d.setVar('BB_NO_NETWORK', '1') 2606 # Fetch again 2607 fetcher.download() 2608 fetcher.unpack(self.unpackdir) 2609 unpackdir = os.path.join(self.unpackdir, 'npm') 2610 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2611 2612 @skipIfNoNpm() 2613 @skipIfNoNetwork() 2614 def test_npm_registry_alternate(self): 2615 url = 'npm://skimdb.npmjs.com;package=@savoirfairelinux/node-server-example;version=1.0.0' 2616 fetcher = bb.fetch.Fetch([url], self.d) 2617 fetcher.download() 2618 fetcher.unpack(self.unpackdir) 2619 unpackdir = os.path.join(self.unpackdir, 'npm') 2620 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2621 2622 @skipIfNoNpm() 2623 @skipIfNoNetwork() 2624 def test_npm_version_latest(self): 2625 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=latest' 2626 fetcher = bb.fetch.Fetch([url], self.d) 2627 fetcher.download() 2628 fetcher.unpack(self.unpackdir) 2629 unpackdir = os.path.join(self.unpackdir, 'npm') 2630 self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) 2631 2632 @skipIfNoNpm() 2633 @skipIfNoNetwork() 2634 def test_npm_registry_invalid(self): 2635 url = 'npm://registry.invalid.org;package=@savoirfairelinux/node-server-example;version=1.0.0' 2636 fetcher = bb.fetch.Fetch([url], self.d) 2637 with self.assertRaises(bb.fetch2.FetchError): 2638 fetcher.download() 2639 2640 @skipIfNoNpm() 2641 @skipIfNoNetwork() 2642 def test_npm_package_invalid(self): 2643 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/invalid;version=1.0.0' 2644 fetcher = bb.fetch.Fetch([url], self.d) 2645 with self.assertRaises(bb.fetch2.FetchError): 2646 fetcher.download() 2647 2648 @skipIfNoNpm() 2649 @skipIfNoNetwork() 2650 def test_npm_version_invalid(self): 2651 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=invalid' 2652 with self.assertRaises(bb.fetch2.ParameterError): 2653 fetcher = bb.fetch.Fetch([url], self.d) 2654 2655 @skipIfNoNpm() 2656 @skipIfNoNetwork() 2657 def test_npm_registry_none(self): 2658 url = 'npm://;package=@savoirfairelinux/node-server-example;version=1.0.0' 2659 with self.assertRaises(bb.fetch2.MalformedUrl): 2660 fetcher = bb.fetch.Fetch([url], self.d) 2661 2662 @skipIfNoNpm() 2663 @skipIfNoNetwork() 2664 def test_npm_package_none(self): 2665 url = 'npm://registry.npmjs.org;version=1.0.0' 2666 with self.assertRaises(bb.fetch2.MissingParameterError): 2667 fetcher = bb.fetch.Fetch([url], self.d) 2668 2669 @skipIfNoNpm() 2670 @skipIfNoNetwork() 2671 def test_npm_version_none(self): 2672 url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example' 2673 with self.assertRaises(bb.fetch2.MissingParameterError): 2674 fetcher = bb.fetch.Fetch([url], self.d) 2675 2676 def create_shrinkwrap_file(self, data): 2677 import json 2678 datadir = os.path.join(self.tempdir, 'data') 2679 swfile = os.path.join(datadir, 'npm-shrinkwrap.json') 2680 bb.utils.mkdirhier(datadir) 2681 with open(swfile, 'w') as f: 2682 json.dump(data, f) 2683 # Also configure the S directory 2684 self.sdir = os.path.join(self.unpackdir, 'S') 2685 self.d.setVar('S', self.sdir) 2686 return swfile 2687 2688 @skipIfNoNpm() 2689 @skipIfNoNetwork() 2690 def test_npmsw(self): 2691 swfile = self.create_shrinkwrap_file({ 2692 'dependencies': { 2693 'array-flatten': { 2694 'version': '1.1.1', 2695 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2696 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=', 2697 'dependencies': { 2698 'content-type': { 2699 'version': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', 2700 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', 2701 'dependencies': { 2702 'cookie': { 2703 'version': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09', 2704 'from': 'git+https://github.com/jshttp/cookie.git' 2705 } 2706 } 2707 } 2708 } 2709 } 2710 } 2711 }) 2712 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2713 fetcher.download() 2714 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2715 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2716 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) 2717 fetcher.unpack(self.unpackdir) 2718 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'npm-shrinkwrap.json'))) 2719 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json'))) 2720 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'package.json'))) 2721 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'node_modules', 'cookie', 'package.json'))) 2722 2723 @skipIfNoNpm() 2724 @skipIfNoNetwork() 2725 def test_npmsw_git(self): 2726 swfile = self.create_shrinkwrap_file({ 2727 'dependencies': { 2728 'cookie': { 2729 'version': 'github:jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09', 2730 'from': 'github:jshttp/cookie.git' 2731 } 2732 } 2733 }) 2734 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2735 fetcher.download() 2736 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) 2737 2738 swfile = self.create_shrinkwrap_file({ 2739 'dependencies': { 2740 'cookie': { 2741 'version': 'jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09', 2742 'from': 'jshttp/cookie.git' 2743 } 2744 } 2745 }) 2746 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2747 fetcher.download() 2748 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) 2749 2750 swfile = self.create_shrinkwrap_file({ 2751 'dependencies': { 2752 'nodejs': { 2753 'version': 'gitlab:gitlab-examples/nodejs.git#892a1f16725e56cc3a2cb0d677be42935c8fc262', 2754 'from': 'gitlab:gitlab-examples/nodejs' 2755 } 2756 } 2757 }) 2758 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2759 fetcher.download() 2760 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'gitlab.com.gitlab-examples.nodejs.git'))) 2761 2762 @skipIfNoNpm() 2763 @skipIfNoNetwork() 2764 def test_npmsw_dev(self): 2765 swfile = self.create_shrinkwrap_file({ 2766 'dependencies': { 2767 'array-flatten': { 2768 'version': '1.1.1', 2769 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2770 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2771 }, 2772 'content-type': { 2773 'version': '1.0.4', 2774 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', 2775 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', 2776 'dev': True 2777 } 2778 } 2779 }) 2780 # Fetch with dev disabled 2781 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2782 fetcher.download() 2783 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2784 self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2785 # Fetch with dev enabled 2786 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';dev=1'], self.d) 2787 fetcher.download() 2788 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) 2789 self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) 2790 2791 @skipIfNoNpm() 2792 @skipIfNoNetwork() 2793 def test_npmsw_destsuffix(self): 2794 swfile = self.create_shrinkwrap_file({ 2795 'dependencies': { 2796 'array-flatten': { 2797 'version': '1.1.1', 2798 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2799 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2800 } 2801 } 2802 }) 2803 fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';destsuffix=foo/bar'], self.d) 2804 fetcher.download() 2805 fetcher.unpack(self.unpackdir) 2806 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'foo', 'bar', 'node_modules', 'array-flatten', 'package.json'))) 2807 2808 def test_npmsw_no_network_no_tarball(self): 2809 swfile = self.create_shrinkwrap_file({ 2810 'dependencies': { 2811 'array-flatten': { 2812 'version': '1.1.1', 2813 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2814 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2815 } 2816 } 2817 }) 2818 self.d.setVar('BB_NO_NETWORK', '1') 2819 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2820 with self.assertRaises(bb.fetch2.NetworkAccess): 2821 fetcher.download() 2822 2823 @skipIfNoNpm() 2824 @skipIfNoNetwork() 2825 def test_npmsw_no_network_with_tarball(self): 2826 # Fetch once to get a tarball 2827 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 2828 fetcher.download() 2829 # Disable network access 2830 self.d.setVar('BB_NO_NETWORK', '1') 2831 # Fetch again 2832 swfile = self.create_shrinkwrap_file({ 2833 'dependencies': { 2834 'array-flatten': { 2835 'version': '1.1.1', 2836 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2837 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2838 } 2839 } 2840 }) 2841 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2842 fetcher.download() 2843 fetcher.unpack(self.unpackdir) 2844 self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json'))) 2845 2846 @skipIfNoNpm() 2847 @skipIfNoNetwork() 2848 def test_npmsw_npm_reusability(self): 2849 # Fetch once with npmsw 2850 swfile = self.create_shrinkwrap_file({ 2851 'dependencies': { 2852 'array-flatten': { 2853 'version': '1.1.1', 2854 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2855 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2856 } 2857 } 2858 }) 2859 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2860 fetcher.download() 2861 # Disable network access 2862 self.d.setVar('BB_NO_NETWORK', '1') 2863 # Fetch again with npm 2864 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 2865 fetcher.download() 2866 fetcher.unpack(self.unpackdir) 2867 self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm', 'package.json'))) 2868 2869 @skipIfNoNpm() 2870 @skipIfNoNetwork() 2871 def test_npmsw_bad_checksum(self): 2872 # Try to fetch with bad checksum 2873 swfile = self.create_shrinkwrap_file({ 2874 'dependencies': { 2875 'array-flatten': { 2876 'version': '1.1.1', 2877 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2878 'integrity': 'sha1-gfNEp2hqgLTFKT6P3AsBYMgsBqg=' 2879 } 2880 } 2881 }) 2882 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2883 with self.assertRaises(bb.fetch2.FetchError): 2884 fetcher.download() 2885 # Fetch correctly to get a tarball 2886 swfile = self.create_shrinkwrap_file({ 2887 'dependencies': { 2888 'array-flatten': { 2889 'version': '1.1.1', 2890 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2891 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2892 } 2893 } 2894 }) 2895 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2896 fetcher.download() 2897 localpath = os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz') 2898 self.assertTrue(os.path.exists(localpath)) 2899 # Modify the tarball 2900 bad = b'bad checksum' 2901 with open(localpath, 'wb') as f: 2902 f.write(bad) 2903 # Verify that the tarball is fetched again 2904 fetcher.download() 2905 badsum = hashlib.sha1(bad).hexdigest() 2906 self.assertTrue(os.path.exists(localpath + '_bad-checksum_' + badsum)) 2907 self.assertTrue(os.path.exists(localpath)) 2908 2909 @skipIfNoNpm() 2910 @skipIfNoNetwork() 2911 def test_npmsw_premirrors(self): 2912 # Fetch once to get a tarball 2913 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 2914 ud = fetcher.ud[fetcher.urls[0]] 2915 fetcher.download() 2916 self.assertTrue(os.path.exists(ud.localpath)) 2917 # Setup the mirror 2918 mirrordir = os.path.join(self.tempdir, 'mirror') 2919 bb.utils.mkdirhier(mirrordir) 2920 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 2921 self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 2922 self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') 2923 # Fetch again 2924 self.assertFalse(os.path.exists(ud.localpath)) 2925 swfile = self.create_shrinkwrap_file({ 2926 'dependencies': { 2927 'array-flatten': { 2928 'version': '1.1.1', 2929 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', 2930 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2931 } 2932 } 2933 }) 2934 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2935 fetcher.download() 2936 self.assertTrue(os.path.exists(ud.localpath)) 2937 2938 @skipIfNoNpm() 2939 @skipIfNoNetwork() 2940 def test_npmsw_mirrors(self): 2941 # Fetch once to get a tarball 2942 fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) 2943 ud = fetcher.ud[fetcher.urls[0]] 2944 fetcher.download() 2945 self.assertTrue(os.path.exists(ud.localpath)) 2946 # Setup the mirror 2947 mirrordir = os.path.join(self.tempdir, 'mirror') 2948 bb.utils.mkdirhier(mirrordir) 2949 os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) 2950 self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir) 2951 # Fetch again with invalid url 2952 self.assertFalse(os.path.exists(ud.localpath)) 2953 swfile = self.create_shrinkwrap_file({ 2954 'dependencies': { 2955 'array-flatten': { 2956 'version': '1.1.1', 2957 'resolved': 'https://invalid', 2958 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' 2959 } 2960 } 2961 }) 2962 fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) 2963 fetcher.download() 2964 self.assertTrue(os.path.exists(ud.localpath)) 2965 2966class GitSharedTest(FetcherTest): 2967 def setUp(self): 2968 super(GitSharedTest, self).setUp() 2969 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master" 2970 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 2971 self.d.setVar("__BBSRCREV_SEEN", "1") 2972 2973 @skipIfNoNetwork() 2974 def test_shared_unpack(self): 2975 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 2976 2977 fetcher.download() 2978 fetcher.unpack(self.unpackdir) 2979 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 2980 self.assertTrue(os.path.exists(alt)) 2981 2982 @skipIfNoNetwork() 2983 def test_noshared_unpack(self): 2984 self.d.setVar('BB_GIT_NOSHARED', '1') 2985 self.unpackdir += '_noshared' 2986 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 2987 2988 fetcher.download() 2989 fetcher.unpack(self.unpackdir) 2990 alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates') 2991 self.assertFalse(os.path.exists(alt)) 2992 2993 2994class FetchPremirroronlyLocalTest(FetcherTest): 2995 2996 def setUp(self): 2997 super(FetchPremirroronlyLocalTest, self).setUp() 2998 self.mirrordir = os.path.join(self.tempdir, "mirrors") 2999 os.mkdir(self.mirrordir) 3000 self.reponame = "bitbake" 3001 self.gitdir = os.path.join(self.tempdir, "git", self.reponame) 3002 self.recipe_url = "git://git.fake.repo/bitbake;branch=master" 3003 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3004 self.d.setVar("BB_NO_NETWORK", "1") 3005 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3006 3007 def make_git_repo(self): 3008 self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz" 3009 recipeurl = "git:/git.fake.repo/bitbake" 3010 os.makedirs(self.gitdir) 3011 self.git("init", self.gitdir) 3012 for i in range(0): 3013 self.git_new_commit() 3014 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3015 3016 def git_new_commit(self): 3017 import random 3018 testfilename = "bibake-fetch.test" 3019 os.unlink(os.path.join(self.mirrordir, self.mirrorname)) 3020 with open(os.path.join(self.gitdir, testfilename), "w") as testfile: 3021 testfile.write("Useless random data {}".format(random.random())) 3022 self.git("add {}".format(testfilename), self.gitdir) 3023 self.git("commit -a -m \"This random commit {}. I'm useless.\"".format(random.random()), self.gitdir) 3024 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3025 return self.git("rev-parse HEAD", self.gitdir).strip() 3026 3027 def test_mirror_commit_nonexistent(self): 3028 self.make_git_repo() 3029 self.d.setVar("SRCREV", "0"*40) 3030 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3031 with self.assertRaises(bb.fetch2.NetworkAccess): 3032 fetcher.download() 3033 3034 def test_mirror_commit_exists(self): 3035 self.make_git_repo() 3036 self.d.setVar("SRCREV", self.git_new_commit()) 3037 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3038 fetcher.download() 3039 fetcher.unpack(self.unpackdir) 3040 3041 def test_mirror_tarball_nonexistent(self): 3042 self.d.setVar("SRCREV", "0"*40) 3043 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3044 with self.assertRaises(bb.fetch2.NetworkAccess): 3045 fetcher.download() 3046 3047class FetchPremirroronlyNetworkTest(FetcherTest): 3048 3049 def setUp(self): 3050 super(FetchPremirroronlyNetworkTest, self).setUp() 3051 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3052 os.mkdir(self.mirrordir) 3053 self.reponame = "fstests" 3054 self.clonedir = os.path.join(self.tempdir, "git") 3055 self.gitdir = os.path.join(self.tempdir, "git", "{}.git".format(self.reponame)) 3056 self.recipe_url = "git://git.yoctoproject.org/fstests" 3057 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3058 self.d.setVar("BB_NO_NETWORK", "0") 3059 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3060 3061 def make_git_repo(self): 3062 import shutil 3063 self.mirrorname = "git2_git.yoctoproject.org.fstests.tar.gz" 3064 os.makedirs(self.clonedir) 3065 self.git("clone --bare --shallow-since=\"01.01.2013\" {}".format(self.recipe_url), self.clonedir) 3066 bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd = self.gitdir) 3067 shutil.rmtree(self.clonedir) 3068 3069 @skipIfNoNetwork() 3070 def test_mirror_tarball_updated(self): 3071 self.make_git_repo() 3072 ## Upstream commit is in the mirror 3073 self.d.setVar("SRCREV", "49d65d53c2bf558ae6e9185af0f3af7b79d255ec") 3074 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3075 fetcher.download() 3076 3077 @skipIfNoNetwork() 3078 def test_mirror_tarball_outdated(self): 3079 self.make_git_repo() 3080 ## Upstream commit not in the mirror 3081 self.d.setVar("SRCREV", "15413486df1f5a5b5af699b6f3ba5f0984e52a9f") 3082 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3083 with self.assertRaises(bb.fetch2.NetworkAccess): 3084 fetcher.download() 3085 3086class FetchPremirroronlyMercurialTest(FetcherTest): 3087 """ Test for premirrors with mercurial repos 3088 the test covers also basic hg:// clone (see fetch_and_create_tarball 3089 """ 3090 def skipIfNoHg(): 3091 import shutil 3092 if not shutil.which('hg'): 3093 return unittest.skip('Mercurial not installed') 3094 return lambda f: f 3095 3096 def setUp(self): 3097 super(FetchPremirroronlyMercurialTest, self).setUp() 3098 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3099 os.mkdir(self.mirrordir) 3100 self.reponame = "libgnt" 3101 self.clonedir = os.path.join(self.tempdir, "hg") 3102 self.recipe_url = "hg://keep.imfreedom.org/libgnt;module=libgnt" 3103 self.d.setVar("SRCREV", "53e8b422faaf") 3104 self.mirrorname = "hg_libgnt_keep.imfreedom.org_.libgnt.tar.gz" 3105 3106 def fetch_and_create_tarball(self): 3107 """ 3108 Ask bitbake to download repo and prepare mirror tarball for us 3109 """ 3110 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1") 3111 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3112 fetcher.download() 3113 mirrorfile = os.path.join(self.d.getVar("DL_DIR"), self.mirrorname) 3114 self.assertTrue(os.path.exists(mirrorfile), "Mirror tarball {} has not been created".format(mirrorfile)) 3115 ## moving tarball to mirror directory 3116 os.rename(mirrorfile, os.path.join(self.mirrordir, self.mirrorname)) 3117 self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "0") 3118 3119 3120 @skipIfNoNetwork() 3121 @skipIfNoHg() 3122 def test_premirror_mercurial(self): 3123 self.fetch_and_create_tarball() 3124 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3125 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3126 self.d.setVar("BB_NO_NETWORK", "1") 3127 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3128 fetcher.download() 3129 3130class FetchPremirroronlyBrokenTarball(FetcherTest): 3131 3132 def setUp(self): 3133 super(FetchPremirroronlyBrokenTarball, self).setUp() 3134 self.mirrordir = os.path.join(self.tempdir, "mirrors") 3135 os.mkdir(self.mirrordir) 3136 self.reponame = "bitbake" 3137 self.gitdir = os.path.join(self.tempdir, "git", self.reponame) 3138 self.recipe_url = "git://git.fake.repo/bitbake" 3139 self.d.setVar("BB_FETCH_PREMIRRORONLY", "1") 3140 self.d.setVar("BB_NO_NETWORK", "1") 3141 self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n") 3142 self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz" 3143 with open(os.path.join(self.mirrordir, self.mirrorname), 'w') as targz: 3144 targz.write("This is not tar.gz file!") 3145 3146 def test_mirror_broken_download(self): 3147 import sys 3148 self.d.setVar("SRCREV", "0"*40) 3149 fetcher = bb.fetch.Fetch([self.recipe_url], self.d) 3150 with self.assertRaises(bb.fetch2.FetchError): 3151 fetcher.download() 3152 stdout = sys.stdout.getvalue() 3153 self.assertFalse(" not a git repository (or any parent up to mount point /)" in stdout) 3154