1# 2# Copyright OpenEmbedded Contributors 3# 4# SPDX-License-Identifier: GPL-2.0-only 5# 6import os 7import subprocess 8import bb 9 10# For reproducible builds, this code sets the default SOURCE_DATE_EPOCH in each 11# component's build environment. The format is number of seconds since the 12# system epoch. 13# 14# Upstream components (generally) respect this environment variable, 15# using it in place of the "current" date and time. 16# See https://reproducible-builds.org/specs/source-date-epoch/ 17# 18# The default value of SOURCE_DATE_EPOCH comes from the function 19# get_source_date_epoch_value which reads from the SDE_FILE, or if the file 20# is not available will use the fallback of SOURCE_DATE_EPOCH_FALLBACK. 21# 22# The SDE_FILE is normally constructed from the function 23# create_source_date_epoch_stamp which is typically added as a postfuncs to 24# the do_unpack task. If a recipe does NOT have do_unpack, it should be added 25# to a task that runs after the source is available and before the 26# do_deploy_source_date_epoch task is executed. 27# 28# If a recipe wishes to override the default behavior it should set it's own 29# SOURCE_DATE_EPOCH or override the do_deploy_source_date_epoch_stamp task 30# with recipe-specific functionality to write the appropriate 31# SOURCE_DATE_EPOCH into the SDE_FILE. 32# 33# SOURCE_DATE_EPOCH is intended to be a reproducible value. This value should 34# be reproducible for anyone who builds the same revision from the same 35# sources. 36# 37# There are 4 ways the create_source_date_epoch_stamp function determines what 38# becomes SOURCE_DATE_EPOCH: 39# 40# 1. Use the value from __source_date_epoch.txt file if this file exists. 41# This file was most likely created in the previous build by one of the 42# following methods 2,3,4. 43# Alternatively, it can be provided by a recipe via SRC_URI. 44# 45# If the file does not exist: 46# 47# 2. If there is a git checkout, use the last git commit timestamp. 48# Git does not preserve file timestamps on checkout. 49# 50# 3. Use the mtime of "known" files such as NEWS, CHANGELOG, ... 51# This works for well-kept repositories distributed via tarball. 52# 53# 4. Use the modification time of the youngest file in the source tree, if 54# there is one. 55# This will be the newest file from the distribution tarball, if any. 56# 57# 5. Fall back to a fixed timestamp (SOURCE_DATE_EPOCH_FALLBACK). 58# 59# Once the value is determined, it is stored in the recipe's SDE_FILE. 60 61def get_source_date_epoch_from_known_files(d, sourcedir): 62 source_date_epoch = None 63 newest_file = None 64 known_files = set(["NEWS", "ChangeLog", "Changelog", "CHANGES"]) 65 for file in known_files: 66 filepath = os.path.join(sourcedir, file) 67 if os.path.isfile(filepath): 68 mtime = int(os.lstat(filepath).st_mtime) 69 # There may be more than one "known_file" present, if so, use the youngest one 70 if not source_date_epoch or mtime > source_date_epoch: 71 source_date_epoch = mtime 72 newest_file = filepath 73 if newest_file: 74 bb.debug(1, "SOURCE_DATE_EPOCH taken from: %s" % newest_file) 75 return source_date_epoch 76 77def find_git_folder(d, sourcedir): 78 # First guess: UNPACKDIR/git 79 # This is the default git fetcher unpack path 80 unpackdir = d.getVar('UNPACKDIR') 81 gitpath = os.path.join(unpackdir, "git/.git") 82 if os.path.isdir(gitpath): 83 return gitpath 84 85 # Second guess: ${S} 86 gitpath = os.path.join(sourcedir, ".git") 87 if os.path.isdir(gitpath): 88 return gitpath 89 90 # Perhaps there was a subpath or destsuffix specified. 91 # Go looking in the UNPACKDIR 92 for root, dirs, files in os.walk(unpackdir, topdown=True): 93 if '.git' in dirs: 94 return os.path.join(root, ".git") 95 96 for root, dirs, files in os.walk(sourcedir, topdown=True): 97 if '.git' in dirs: 98 return os.path.join(root, ".git") 99 100 bb.warn("Failed to find a git repository in UNPACKDIR: %s" % unpackdir) 101 return None 102 103def get_source_date_epoch_from_git(d, sourcedir): 104 if not "git://" in d.getVar('SRC_URI') and not "gitsm://" in d.getVar('SRC_URI'): 105 return None 106 107 gitpath = find_git_folder(d, sourcedir) 108 if not gitpath: 109 return None 110 111 # Check that the repository has a valid HEAD; it may not if subdir is used 112 # in SRC_URI 113 p = subprocess.run(['git', '--git-dir', gitpath, 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 114 if p.returncode != 0: 115 bb.debug(1, "%s does not have a valid HEAD: %s" % (gitpath, p.stdout.decode('utf-8'))) 116 return None 117 118 bb.debug(1, "git repository: %s" % gitpath) 119 p = subprocess.run(['git', '-c', 'log.showSignature=false', '--git-dir', gitpath, 'log', '-1', '--pretty=%ct'], 120 check=True, stdout=subprocess.PIPE) 121 return int(p.stdout.decode('utf-8')) 122 123def get_source_date_epoch_from_youngest_file(d, sourcedir): 124 if sourcedir == d.getVar('UNPACKDIR'): 125 # These sources are almost certainly not from a tarball 126 return None 127 128 # Do it the hard way: check all files and find the youngest one... 129 source_date_epoch = None 130 newest_file = None 131 for root, dirs, files in os.walk(sourcedir, topdown=True): 132 files = [f for f in files if not f[0] == '.'] 133 134 for fname in files: 135 if fname == "singletask.lock": 136 # Ignore externalsrc/devtool lockfile [YOCTO #14921] 137 continue 138 filename = os.path.join(root, fname) 139 try: 140 mtime = int(os.lstat(filename).st_mtime) 141 except ValueError: 142 mtime = 0 143 if not source_date_epoch or mtime > source_date_epoch: 144 source_date_epoch = mtime 145 newest_file = filename 146 147 if newest_file: 148 bb.debug(1, "Newest file found: %s" % newest_file) 149 return source_date_epoch 150 151def fixed_source_date_epoch(d): 152 bb.debug(1, "No tarball or git repo found to determine SOURCE_DATE_EPOCH") 153 source_date_epoch = d.getVar('SOURCE_DATE_EPOCH_FALLBACK') 154 if source_date_epoch: 155 bb.debug(1, "Using SOURCE_DATE_EPOCH_FALLBACK") 156 return int(source_date_epoch) 157 return 0 158 159def get_source_date_epoch(d, sourcedir): 160 return ( 161 get_source_date_epoch_from_git(d, sourcedir) or 162 get_source_date_epoch_from_youngest_file(d, sourcedir) or 163 fixed_source_date_epoch(d) # Last resort 164 ) 165 166def epochfile_read(epochfile, d): 167 cached, efile = d.getVar('__CACHED_SOURCE_DATE_EPOCH') or (None, None) 168 if cached and efile == epochfile: 169 return cached 170 171 if cached and epochfile != efile: 172 bb.debug(1, "Epoch file changed from %s to %s" % (efile, epochfile)) 173 174 source_date_epoch = int(d.getVar('SOURCE_DATE_EPOCH_FALLBACK')) 175 try: 176 with open(epochfile, 'r') as f: 177 s = f.read() 178 try: 179 source_date_epoch = int(s) 180 except ValueError: 181 bb.warn("SOURCE_DATE_EPOCH value '%s' is invalid. Reverting to SOURCE_DATE_EPOCH_FALLBACK" % s) 182 source_date_epoch = int(d.getVar('SOURCE_DATE_EPOCH_FALLBACK')) 183 bb.debug(1, "SOURCE_DATE_EPOCH: %d" % source_date_epoch) 184 except FileNotFoundError: 185 bb.debug(1, "Cannot find %s. SOURCE_DATE_EPOCH will default to %d" % (epochfile, source_date_epoch)) 186 187 d.setVar('__CACHED_SOURCE_DATE_EPOCH', (str(source_date_epoch), epochfile)) 188 return str(source_date_epoch) 189 190def epochfile_write(source_date_epoch, epochfile, d): 191 192 bb.debug(1, "SOURCE_DATE_EPOCH: %d" % source_date_epoch) 193 bb.utils.mkdirhier(os.path.dirname(epochfile)) 194 195 tmp_file = "%s.new" % epochfile 196 with open(tmp_file, 'w') as f: 197 f.write(str(source_date_epoch)) 198 os.rename(tmp_file, epochfile) 199