1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6 7import errno 8import glob 9import shutil 10import subprocess 11import os.path 12 13def join(*paths): 14 """Like os.path.join but doesn't treat absolute RHS specially""" 15 return os.path.normpath("/".join(paths)) 16 17def relative(src, dest): 18 """ Return a relative path from src to dest. 19 20 >>> relative("/usr/bin", "/tmp/foo/bar") 21 ../../tmp/foo/bar 22 23 >>> relative("/usr/bin", "/usr/lib") 24 ../lib 25 26 >>> relative("/tmp", "/tmp/foo/bar") 27 foo/bar 28 """ 29 30 return os.path.relpath(dest, src) 31 32def make_relative_symlink(path): 33 """ Convert an absolute symlink to a relative one """ 34 if not os.path.islink(path): 35 return 36 link = os.readlink(path) 37 if not os.path.isabs(link): 38 return 39 40 # find the common ancestor directory 41 ancestor = path 42 depth = 0 43 while ancestor and not link.startswith(ancestor): 44 ancestor = ancestor.rpartition('/')[0] 45 depth += 1 46 47 if not ancestor: 48 print("make_relative_symlink() Error: unable to find the common ancestor of %s and its target" % path) 49 return 50 51 base = link.partition(ancestor)[2].strip('/') 52 while depth > 1: 53 base = "../" + base 54 depth -= 1 55 56 os.remove(path) 57 os.symlink(base, path) 58 59def replace_absolute_symlinks(basedir, d): 60 """ 61 Walk basedir looking for absolute symlinks and replacing them with relative ones. 62 The absolute links are assumed to be relative to basedir 63 (compared to make_relative_symlink above which tries to compute common ancestors 64 using pattern matching instead) 65 """ 66 for walkroot, dirs, files in os.walk(basedir): 67 for file in files + dirs: 68 path = os.path.join(walkroot, file) 69 if not os.path.islink(path): 70 continue 71 link = os.readlink(path) 72 if not os.path.isabs(link): 73 continue 74 walkdir = os.path.dirname(path.rpartition(basedir)[2]) 75 base = os.path.relpath(link, walkdir) 76 bb.debug(2, "Replacing absolute path %s with relative path %s" % (link, base)) 77 os.remove(path) 78 os.symlink(base, path) 79 80def format_display(path, metadata): 81 """ Prepare a path for display to the user. """ 82 rel = relative(metadata.getVar("TOPDIR"), path) 83 if len(rel) > len(path): 84 return path 85 else: 86 return rel 87 88def copytree(src, dst): 89 # We could use something like shutil.copytree here but it turns out to 90 # to be slow. It takes twice as long copying to an empty directory. 91 # If dst already has contents performance can be 15 time slower 92 # This way we also preserve hardlinks between files in the tree. 93 94 bb.utils.mkdirhier(dst) 95 cmd = "tar --xattrs --xattrs-include='*' -cf - -S -C %s -p . | tar --xattrs --xattrs-include='*' -xf - -C %s" % (src, dst) 96 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) 97 98def copyhardlinktree(src, dst): 99 """Make a tree of hard links when possible, otherwise copy.""" 100 bb.utils.mkdirhier(dst) 101 if os.path.isdir(src) and not len(os.listdir(src)): 102 return 103 104 canhard = False 105 testfile = None 106 for root, dirs, files in os.walk(src): 107 if len(files): 108 testfile = os.path.join(root, files[0]) 109 break 110 111 if testfile is not None: 112 try: 113 os.link(testfile, os.path.join(dst, 'testfile')) 114 os.unlink(os.path.join(dst, 'testfile')) 115 canhard = True 116 except Exception as e: 117 bb.debug(2, "Hardlink test failed with " + str(e)) 118 119 if (canhard): 120 # Need to copy directories only with tar first since cp will error if two 121 # writers try and create a directory at the same time 122 cmd = "cd %s; find . -type d -print | tar --xattrs --xattrs-include='*' -cf - -S -C %s -p --no-recursion --files-from - | tar --xattrs --xattrs-include='*' -xhf - -C %s" % (src, src, dst) 123 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) 124 source = '' 125 if os.path.isdir(src): 126 if len(glob.glob('%s/.??*' % src)) > 0: 127 source = './.??* ' 128 if len(glob.glob('%s/**' % src)) > 0: 129 source += './*' 130 s_dir = src 131 else: 132 source = src 133 s_dir = os.getcwd() 134 cmd = 'cp -afl --preserve=xattr %s %s' % (source, os.path.realpath(dst)) 135 subprocess.check_output(cmd, shell=True, cwd=s_dir, stderr=subprocess.STDOUT) 136 else: 137 copytree(src, dst) 138 139def copyhardlink(src, dst): 140 """Make a hard link when possible, otherwise copy.""" 141 142 try: 143 os.link(src, dst) 144 except OSError: 145 shutil.copy(src, dst) 146 147def remove(path, recurse=True): 148 """ 149 Equivalent to rm -f or rm -rf 150 NOTE: be careful about passing paths that may contain filenames with 151 wildcards in them (as opposed to passing an actual wildcarded path) - 152 since we use glob.glob() to expand the path. Filenames containing 153 square brackets are particularly problematic since the they may not 154 actually expand to match the original filename. 155 """ 156 for name in glob.glob(path): 157 try: 158 os.unlink(name) 159 except OSError as exc: 160 if recurse and exc.errno == errno.EISDIR: 161 shutil.rmtree(name) 162 elif exc.errno != errno.ENOENT: 163 raise 164 165def symlink(source, destination, force=False): 166 """Create a symbolic link""" 167 try: 168 if force: 169 remove(destination) 170 os.symlink(source, destination) 171 except OSError as e: 172 if e.errno != errno.EEXIST or os.readlink(destination) != source: 173 raise 174 175def relsymlink(target, name, force=False): 176 symlink(os.path.relpath(target, os.path.dirname(name)), name, force=force) 177 178def find(dir, **walkoptions): 179 """ Given a directory, recurses into that directory, 180 returning all files as absolute paths. """ 181 182 for root, dirs, files in os.walk(dir, **walkoptions): 183 for file in files: 184 yield os.path.join(root, file) 185 186 187## realpath() related functions 188def __is_path_below(file, root): 189 return (file + os.path.sep).startswith(root) 190 191def __realpath_rel(start, rel_path, root, loop_cnt, assume_dir): 192 """Calculates real path of symlink 'start' + 'rel_path' below 193 'root'; no part of 'start' below 'root' must contain symlinks. """ 194 have_dir = True 195 196 for d in rel_path.split(os.path.sep): 197 if not have_dir and not assume_dir: 198 raise OSError(errno.ENOENT, "no such directory %s" % start) 199 200 if d == os.path.pardir: # '..' 201 if len(start) >= len(root): 202 # do not follow '..' before root 203 start = os.path.dirname(start) 204 else: 205 # emit warning? 206 pass 207 else: 208 (start, have_dir) = __realpath(os.path.join(start, d), 209 root, loop_cnt, assume_dir) 210 211 assert(__is_path_below(start, root)) 212 213 return start 214 215def __realpath(file, root, loop_cnt, assume_dir): 216 while os.path.islink(file) and len(file) >= len(root): 217 if loop_cnt == 0: 218 raise OSError(errno.ELOOP, file) 219 220 loop_cnt -= 1 221 target = os.path.normpath(os.readlink(file)) 222 223 if not os.path.isabs(target): 224 tdir = os.path.dirname(file) 225 assert(__is_path_below(tdir, root)) 226 else: 227 tdir = root 228 229 file = __realpath_rel(tdir, target, root, loop_cnt, assume_dir) 230 231 try: 232 is_dir = os.path.isdir(file) 233 except: 234 is_dir = false 235 236 return (file, is_dir) 237 238def realpath(file, root, use_physdir = True, loop_cnt = 100, assume_dir = False): 239 """ Returns the canonical path of 'file' with assuming a 240 toplevel 'root' directory. When 'use_physdir' is set, all 241 preceding path components of 'file' will be resolved first; 242 this flag should be set unless it is guaranteed that there is 243 no symlink in the path. When 'assume_dir' is not set, missing 244 path components will raise an ENOENT error""" 245 246 root = os.path.normpath(root) 247 file = os.path.normpath(file) 248 249 if not root.endswith(os.path.sep): 250 # letting root end with '/' makes some things easier 251 root = root + os.path.sep 252 253 if not __is_path_below(file, root): 254 raise OSError(errno.EINVAL, "file '%s' is not below root" % file) 255 256 try: 257 if use_physdir: 258 file = __realpath_rel(root, file[(len(root) - 1):], root, loop_cnt, assume_dir) 259 else: 260 file = __realpath(file, root, loop_cnt, assume_dir)[0] 261 except OSError as e: 262 if e.errno == errno.ELOOP: 263 # make ELOOP more readable; without catching it, there will 264 # be printed a backtrace with 100s of OSError exceptions 265 # else 266 raise OSError(errno.ELOOP, 267 "too much recursions while resolving '%s'; loop in '%s'" % 268 (file, e.strerror)) 269 270 raise 271 272 return file 273 274def is_path_parent(possible_parent, *paths): 275 """ 276 Return True if a path is the parent of another, False otherwise. 277 Multiple paths to test can be specified in which case all 278 specified test paths must be under the parent in order to 279 return True. 280 """ 281 def abs_path_trailing(pth): 282 pth_abs = os.path.abspath(pth) 283 if not pth_abs.endswith(os.sep): 284 pth_abs += os.sep 285 return pth_abs 286 287 possible_parent_abs = abs_path_trailing(possible_parent) 288 if not paths: 289 return False 290 for path in paths: 291 path_abs = abs_path_trailing(path) 292 if not path_abs.startswith(possible_parent_abs): 293 return False 294 return True 295 296def which_wild(pathname, path=None, mode=os.F_OK, *, reverse=False, candidates=False): 297 """Search a search path for pathname, supporting wildcards. 298 299 Return all paths in the specific search path matching the wildcard pattern 300 in pathname, returning only the first encountered for each file. If 301 candidates is True, information on all potential candidate paths are 302 included. 303 """ 304 paths = (path or os.environ.get('PATH', os.defpath)).split(':') 305 if reverse: 306 paths.reverse() 307 308 seen, files = set(), [] 309 for index, element in enumerate(paths): 310 if not os.path.isabs(element): 311 element = os.path.abspath(element) 312 313 candidate = os.path.join(element, pathname) 314 globbed = glob.glob(candidate) 315 if globbed: 316 for found_path in sorted(globbed): 317 if not os.access(found_path, mode): 318 continue 319 rel = os.path.relpath(found_path, element) 320 if rel not in seen: 321 seen.add(rel) 322 if candidates: 323 files.append((found_path, [os.path.join(p, rel) for p in paths[:index+1]])) 324 else: 325 files.append(found_path) 326 327 return files 328 329def canonicalize(paths, sep=','): 330 """Given a string with paths (separated by commas by default), expand 331 each path using os.path.realpath() and return the resulting paths as a 332 string (separated using the same separator a the original string). 333 """ 334 # Ignore paths containing "$" as they are assumed to be unexpanded bitbake 335 # variables. Normally they would be ignored, e.g., when passing the paths 336 # through the shell they would expand to empty strings. However, when they 337 # are passed through os.path.realpath(), it will cause them to be prefixed 338 # with the absolute path to the current directory and thus not be empty 339 # anymore. 340 # 341 # Also maintain trailing slashes, as the paths may actually be used as 342 # prefixes in sting compares later on, where the slashes then are important. 343 canonical_paths = [] 344 for path in (paths or '').split(sep): 345 if '$' not in path: 346 trailing_slash = path.endswith('/') and '/' or '' 347 canonical_paths.append(os.path.realpath(path) + trailing_slash) 348 349 return sep.join(canonical_paths) 350