1# Development tool - sdk-update command plugin 2# 3# Copyright (C) 2015-2016 Intel Corporation 4# 5# SPDX-License-Identifier: GPL-2.0-only 6# 7 8import os 9import subprocess 10import logging 11import glob 12import shutil 13import errno 14import sys 15import tempfile 16import re 17from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError 18 19logger = logging.getLogger('devtool') 20 21def parse_locked_sigs(sigfile_path): 22 """Return <pn:task>:<hash> dictionary""" 23 sig_dict = {} 24 with open(sigfile_path) as f: 25 lines = f.readlines() 26 for line in lines: 27 if ':' in line: 28 taskkey, _, hashval = line.rpartition(':') 29 sig_dict[taskkey.strip()] = hashval.split()[0] 30 return sig_dict 31 32def generate_update_dict(sigfile_new, sigfile_old): 33 """Return a dict containing <pn:task>:<hash> which indicates what need to be updated""" 34 update_dict = {} 35 sigdict_new = parse_locked_sigs(sigfile_new) 36 sigdict_old = parse_locked_sigs(sigfile_old) 37 for k in sigdict_new: 38 if k not in sigdict_old: 39 update_dict[k] = sigdict_new[k] 40 continue 41 if sigdict_new[k] != sigdict_old[k]: 42 update_dict[k] = sigdict_new[k] 43 continue 44 return update_dict 45 46def get_sstate_objects(update_dict, sstate_dir): 47 """Return a list containing sstate objects which are to be installed""" 48 sstate_objects = [] 49 for k in update_dict: 50 files = set() 51 hashval = update_dict[k] 52 p = sstate_dir + '/' + hashval[:2] + '/*' + hashval + '*.tgz' 53 files |= set(glob.glob(p)) 54 p = sstate_dir + '/*/' + hashval[:2] + '/*' + hashval + '*.tgz' 55 files |= set(glob.glob(p)) 56 files = list(files) 57 if len(files) == 1: 58 sstate_objects.extend(files) 59 elif len(files) > 1: 60 logger.error("More than one matching sstate object found for %s" % hashval) 61 62 return sstate_objects 63 64def mkdir(d): 65 try: 66 os.makedirs(d) 67 except OSError as e: 68 if e.errno != errno.EEXIST: 69 raise e 70 71def install_sstate_objects(sstate_objects, src_sdk, dest_sdk): 72 """Install sstate objects into destination SDK""" 73 sstate_dir = os.path.join(dest_sdk, 'sstate-cache') 74 if not os.path.exists(sstate_dir): 75 logger.error("Missing sstate-cache directory in %s, it might not be an extensible SDK." % dest_sdk) 76 raise 77 for sb in sstate_objects: 78 dst = sb.replace(src_sdk, dest_sdk) 79 destdir = os.path.dirname(dst) 80 mkdir(destdir) 81 logger.debug("Copying %s to %s" % (sb, dst)) 82 shutil.copy(sb, dst) 83 84def check_manifest(fn, basepath): 85 import bb.utils 86 changedfiles = [] 87 with open(fn, 'r') as f: 88 for line in f: 89 splitline = line.split() 90 if len(splitline) > 1: 91 chksum = splitline[0] 92 fpath = splitline[1] 93 curr_chksum = bb.utils.sha256_file(os.path.join(basepath, fpath)) 94 if chksum != curr_chksum: 95 logger.debug('File %s changed: old csum = %s, new = %s' % (os.path.join(basepath, fpath), curr_chksum, chksum)) 96 changedfiles.append(fpath) 97 return changedfiles 98 99def sdk_update(args, config, basepath, workspace): 100 """Entry point for devtool sdk-update command""" 101 updateserver = args.updateserver 102 if not updateserver: 103 updateserver = config.get('SDK', 'updateserver', '') 104 logger.debug("updateserver: %s" % updateserver) 105 106 # Make sure we are using sdk-update from within SDK 107 logger.debug("basepath = %s" % basepath) 108 old_locked_sig_file_path = os.path.join(basepath, 'conf/locked-sigs.inc') 109 if not os.path.exists(old_locked_sig_file_path): 110 logger.error("Not using devtool's sdk-update command from within an extensible SDK. Please specify correct basepath via --basepath option") 111 return -1 112 else: 113 logger.debug("Found conf/locked-sigs.inc in %s" % basepath) 114 115 if not '://' in updateserver: 116 logger.error("Update server must be a URL") 117 return -1 118 119 layers_dir = os.path.join(basepath, 'layers') 120 conf_dir = os.path.join(basepath, 'conf') 121 122 # Grab variable values 123 tinfoil = setup_tinfoil(config_only=True, basepath=basepath) 124 try: 125 stamps_dir = tinfoil.config_data.getVar('STAMPS_DIR') 126 sstate_mirrors = tinfoil.config_data.getVar('SSTATE_MIRRORS') 127 site_conf_version = tinfoil.config_data.getVar('SITE_CONF_VERSION') 128 finally: 129 tinfoil.shutdown() 130 131 tmpsdk_dir = tempfile.mkdtemp() 132 try: 133 os.makedirs(os.path.join(tmpsdk_dir, 'conf')) 134 new_locked_sig_file_path = os.path.join(tmpsdk_dir, 'conf', 'locked-sigs.inc') 135 # Fetch manifest from server 136 tmpmanifest = os.path.join(tmpsdk_dir, 'conf', 'sdk-conf-manifest') 137 ret = subprocess.call("wget -q -O %s %s/conf/sdk-conf-manifest" % (tmpmanifest, updateserver), shell=True) 138 if ret != 0: 139 logger.error("Cannot dowload files from %s" % updateserver) 140 return ret 141 changedfiles = check_manifest(tmpmanifest, basepath) 142 if not changedfiles: 143 logger.info("Already up-to-date") 144 return 0 145 # Update metadata 146 logger.debug("Updating metadata via git ...") 147 #Check for the status before doing a fetch and reset 148 if os.path.exists(os.path.join(basepath, 'layers/.git')): 149 out = subprocess.check_output("git status --porcelain", shell=True, cwd=layers_dir) 150 if not out: 151 ret = subprocess.call("git fetch --all; git reset --hard @{u}", shell=True, cwd=layers_dir) 152 else: 153 logger.error("Failed to update metadata as there have been changes made to it. Aborting."); 154 logger.error("Changed files:\n%s" % out); 155 return -1 156 else: 157 ret = -1 158 if ret != 0: 159 ret = subprocess.call("git clone %s/layers/.git" % updateserver, shell=True, cwd=tmpsdk_dir) 160 if ret != 0: 161 logger.error("Updating metadata via git failed") 162 return ret 163 logger.debug("Updating conf files ...") 164 for changedfile in changedfiles: 165 ret = subprocess.call("wget -q -O %s %s/%s" % (changedfile, updateserver, changedfile), shell=True, cwd=tmpsdk_dir) 166 if ret != 0: 167 logger.error("Updating %s failed" % changedfile) 168 return ret 169 170 # Check if UNINATIVE_CHECKSUM changed 171 uninative = False 172 if 'conf/local.conf' in changedfiles: 173 def read_uninative_checksums(fn): 174 chksumitems = [] 175 with open(fn, 'r') as f: 176 for line in f: 177 if line.startswith('UNINATIVE_CHECKSUM'): 178 splitline = re.split(r'[\[\]"\']', line) 179 if len(splitline) > 3: 180 chksumitems.append((splitline[1], splitline[3])) 181 return chksumitems 182 183 oldsums = read_uninative_checksums(os.path.join(basepath, 'conf/local.conf')) 184 newsums = read_uninative_checksums(os.path.join(tmpsdk_dir, 'conf/local.conf')) 185 if oldsums != newsums: 186 uninative = True 187 for buildarch, chksum in newsums: 188 uninative_file = os.path.join('downloads', 'uninative', chksum, '%s-nativesdk-libc.tar.bz2' % buildarch) 189 mkdir(os.path.join(tmpsdk_dir, os.path.dirname(uninative_file))) 190 ret = subprocess.call("wget -q -O %s %s/%s" % (uninative_file, updateserver, uninative_file), shell=True, cwd=tmpsdk_dir) 191 192 # Ok, all is well at this point - move everything over 193 tmplayers_dir = os.path.join(tmpsdk_dir, 'layers') 194 if os.path.exists(tmplayers_dir): 195 shutil.rmtree(layers_dir) 196 shutil.move(tmplayers_dir, layers_dir) 197 for changedfile in changedfiles: 198 destfile = os.path.join(basepath, changedfile) 199 os.remove(destfile) 200 shutil.move(os.path.join(tmpsdk_dir, changedfile), destfile) 201 os.remove(os.path.join(conf_dir, 'sdk-conf-manifest')) 202 shutil.move(tmpmanifest, conf_dir) 203 if uninative: 204 shutil.rmtree(os.path.join(basepath, 'downloads', 'uninative')) 205 shutil.move(os.path.join(tmpsdk_dir, 'downloads', 'uninative'), os.path.join(basepath, 'downloads')) 206 207 if not sstate_mirrors: 208 with open(os.path.join(conf_dir, 'site.conf'), 'a') as f: 209 f.write('SCONF_VERSION = "%s"\n' % site_conf_version) 210 f.write('SSTATE_MIRRORS:append = " file://.* %s/sstate-cache/PATH"\n' % updateserver) 211 finally: 212 shutil.rmtree(tmpsdk_dir) 213 214 if not args.skip_prepare: 215 # Find all potentially updateable tasks 216 sdk_update_targets = [] 217 tasks = ['do_populate_sysroot', 'do_packagedata'] 218 for root, _, files in os.walk(stamps_dir): 219 for fn in files: 220 if not '.sigdata.' in fn: 221 for task in tasks: 222 if '.%s.' % task in fn or '.%s_setscene.' % task in fn: 223 sdk_update_targets.append('%s:%s' % (os.path.basename(root), task)) 224 # Run bitbake command for the whole SDK 225 logger.info("Preparing build system... (This may take some time.)") 226 try: 227 exec_build_env_command(config.init_path, basepath, 'bitbake --setscene-only %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT) 228 output, _ = exec_build_env_command(config.init_path, basepath, 'bitbake -n %s' % ' '.join(sdk_update_targets), stderr=subprocess.STDOUT) 229 runlines = [] 230 for line in output.splitlines(): 231 if 'Running task ' in line: 232 runlines.append(line) 233 if runlines: 234 logger.error('Unexecuted tasks found in preparation log:\n %s' % '\n '.join(runlines)) 235 return -1 236 except bb.process.ExecutionError as e: 237 logger.error('Preparation failed:\n%s' % e.stdout) 238 return -1 239 return 0 240 241def sdk_install(args, config, basepath, workspace): 242 """Entry point for the devtool sdk-install command""" 243 244 import oe.recipeutils 245 import bb.process 246 247 for recipe in args.recipename: 248 if recipe in workspace: 249 raise DevtoolError('recipe %s is a recipe in your workspace' % recipe) 250 251 tasks = ['do_populate_sysroot', 'do_packagedata'] 252 stampprefixes = {} 253 def checkstamp(recipe): 254 stampprefix = stampprefixes[recipe] 255 stamps = glob.glob(stampprefix + '*') 256 for stamp in stamps: 257 if '.sigdata.' not in stamp and stamp.startswith((stampprefix + '.', stampprefix + '_setscene.')): 258 return True 259 else: 260 return False 261 262 install_recipes = [] 263 tinfoil = setup_tinfoil(config_only=False, basepath=basepath) 264 try: 265 for recipe in args.recipename: 266 rd = parse_recipe(config, tinfoil, recipe, True) 267 if not rd: 268 return 1 269 stampprefixes[recipe] = '%s.%s' % (rd.getVar('STAMP'), tasks[0]) 270 if checkstamp(recipe): 271 logger.info('%s is already installed' % recipe) 272 else: 273 install_recipes.append(recipe) 274 finally: 275 tinfoil.shutdown() 276 277 if install_recipes: 278 logger.info('Installing %s...' % ', '.join(install_recipes)) 279 install_tasks = [] 280 for recipe in install_recipes: 281 for task in tasks: 282 if recipe.endswith('-native') and 'package' in task: 283 continue 284 install_tasks.append('%s:%s' % (recipe, task)) 285 options = '' 286 if not args.allow_build: 287 options += ' --setscene-only' 288 try: 289 exec_build_env_command(config.init_path, basepath, 'bitbake %s %s' % (options, ' '.join(install_tasks)), watch=True) 290 except bb.process.ExecutionError as e: 291 raise DevtoolError('Failed to install %s:\n%s' % (recipe, str(e))) 292 failed = False 293 for recipe in install_recipes: 294 if checkstamp(recipe): 295 logger.info('Successfully installed %s' % recipe) 296 else: 297 raise DevtoolError('Failed to install %s - unavailable' % recipe) 298 failed = True 299 if failed: 300 return 2 301 302 try: 303 exec_build_env_command(config.init_path, basepath, 'bitbake build-sysroots -c build_native_sysroot', watch=True) 304 exec_build_env_command(config.init_path, basepath, 'bitbake build-sysroots -c build_target_sysroot', watch=True) 305 except bb.process.ExecutionError as e: 306 raise DevtoolError('Failed to bitbake build-sysroots:\n%s' % (str(e))) 307 308 309def register_commands(subparsers, context): 310 """Register devtool subcommands from the sdk plugin""" 311 if context.fixed_setup: 312 parser_sdk = subparsers.add_parser('sdk-update', 313 help='Update SDK components', 314 description='Updates installed SDK components from a remote server', 315 group='sdk') 316 updateserver = context.config.get('SDK', 'updateserver', '') 317 if updateserver: 318 parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from (default %s)' % updateserver, nargs='?') 319 else: 320 parser_sdk.add_argument('updateserver', help='The update server to fetch latest SDK components from') 321 parser_sdk.add_argument('--skip-prepare', action="store_true", help='Skip re-preparing the build system after updating (for debugging only)') 322 parser_sdk.set_defaults(func=sdk_update) 323 324 parser_sdk_install = subparsers.add_parser('sdk-install', 325 help='Install additional SDK components', 326 description='Installs additional recipe development files into the SDK. (You can use "devtool search" to find available recipes.)', 327 group='sdk') 328 parser_sdk_install.add_argument('recipename', help='Name of the recipe to install the development artifacts for', nargs='+') 329 parser_sdk_install.add_argument('-s', '--allow-build', help='Allow building requested item(s) from source', action='store_true') 330 parser_sdk_install.set_defaults(func=sdk_install) 331