1# 2# BitBake Toaster Implementation 3# 4# Copyright (C) 2016-2017 Intel Corporation 5# 6# SPDX-License-Identifier: GPL-2.0-only 7# 8 9from django.core.management.base import BaseCommand 10 11from orm.models import Layer, Release, Layer_Version 12from orm.models import LayerVersionDependency, Machine, Recipe 13from orm.models import Distro 14from orm.models import ToasterSetting 15 16import os 17import sys 18 19import logging 20import threading 21import time 22logger = logging.getLogger("toaster") 23 24DEFAULT_LAYERINDEX_SERVER = "http://layers.openembedded.org/layerindex/api/" 25 26# Add path to bitbake modules for layerindexlib 27# lib/toaster/orm/management/commands/lsupdates.py (abspath) 28# lib/toaster/orm/management/commands (dirname) 29# lib/toaster/orm/management (dirname) 30# lib/toaster/orm (dirname) 31# lib/toaster/ (dirname) 32# lib/ (dirname) 33path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) 34sys.path.insert(0, path) 35 36import layerindexlib 37 38 39class Spinner(threading.Thread): 40 """ A simple progress spinner to indicate download/parsing is happening""" 41 def __init__(self, *args, **kwargs): 42 super(Spinner, self).__init__(*args, **kwargs) 43 self.setDaemon(True) 44 self.signal = True 45 46 def run(self): 47 os.system('setterm -cursor off') 48 while self.signal: 49 for char in ["/", "-", "\\", "|"]: 50 sys.stdout.write("\r" + char) 51 sys.stdout.flush() 52 time.sleep(0.25) 53 os.system('setterm -cursor on') 54 55 def stop(self): 56 self.signal = False 57 58 59class Command(BaseCommand): 60 args = "" 61 help = "Updates locally cached information from a layerindex server" 62 63 def mini_progress(self, what, i, total): 64 i = i + 1 65 pec = (float(i)/float(total))*100 66 67 sys.stdout.write("\rUpdating %s %d%%" % 68 (what, 69 pec)) 70 sys.stdout.flush() 71 if int(pec) == 100: 72 sys.stdout.write("\n") 73 sys.stdout.flush() 74 75 def update(self): 76 """ 77 Fetches layer, recipe and machine information from a layerindex 78 server 79 """ 80 os.system('setterm -cursor off') 81 82 self.apiurl = DEFAULT_LAYERINDEX_SERVER 83 if ToasterSetting.objects.filter(name='CUSTOM_LAYERINDEX_SERVER').count() == 1: 84 self.apiurl = ToasterSetting.objects.get(name = 'CUSTOM_LAYERINDEX_SERVER').value 85 86 assert self.apiurl is not None 87 88 # update branches; only those that we already have names listed in the 89 # Releases table 90 whitelist_branch_names = [rel.branch_name 91 for rel in Release.objects.all()] 92 if len(whitelist_branch_names) == 0: 93 raise Exception("Failed to make list of branches to fetch") 94 95 logger.info("Fetching metadata for %s", 96 " ".join(whitelist_branch_names)) 97 98 # We require a non-empty bb.data, but we can fake it with a dictionary 99 layerindex = layerindexlib.LayerIndex({"DUMMY" : "VALUE"}) 100 101 http_progress = Spinner() 102 http_progress.start() 103 104 if whitelist_branch_names: 105 url_branches = ";branch=%s" % ','.join(whitelist_branch_names) 106 else: 107 url_branches = "" 108 layerindex.load_layerindex("%s%s" % (self.apiurl, url_branches)) 109 110 http_progress.stop() 111 112 # We know we're only processing one entry, so we reference it here 113 # (this is cheating...) 114 index = layerindex.indexes[0] 115 116 # Map the layer index branches to toaster releases 117 li_branch_id_to_toaster_release = {} 118 119 logger.info("Processing releases") 120 121 total = len(index.branches) 122 for i, id in enumerate(index.branches): 123 li_branch_id_to_toaster_release[id] = \ 124 Release.objects.get(name=index.branches[id].name) 125 self.mini_progress("Releases", i, total) 126 127 # keep a track of the layerindex (li) id mappings so that 128 # layer_versions can be created for these layers later on 129 li_layer_id_to_toaster_layer_id = {} 130 131 logger.info("Processing layers") 132 133 total = len(index.layerItems) 134 for i, id in enumerate(index.layerItems): 135 try: 136 l, created = Layer.objects.get_or_create(name=index.layerItems[id].name) 137 l.up_date = index.layerItems[id].updated 138 l.summary = index.layerItems[id].summary 139 l.description = index.layerItems[id].description 140 141 if created: 142 # predefined layers in the fixtures (for example poky.xml) 143 # always preempt the Layer Index for these values 144 l.vcs_url = index.layerItems[id].vcs_url 145 l.vcs_web_url = index.layerItems[id].vcs_web_url 146 l.vcs_web_tree_base_url = index.layerItems[id].vcs_web_tree_base_url 147 l.vcs_web_file_base_url = index.layerItems[id].vcs_web_file_base_url 148 l.save() 149 except Layer.MultipleObjectsReturned: 150 logger.info("Skipped %s as we found multiple layers and " 151 "don't know which to update" % 152 index.layerItems[id].name) 153 154 li_layer_id_to_toaster_layer_id[id] = l.pk 155 156 self.mini_progress("layers", i, total) 157 158 # update layer_versions 159 logger.info("Processing layer versions") 160 161 # Map Layer index layer_branch object id to 162 # layer_version toaster object id 163 li_layer_branch_id_to_toaster_lv_id = {} 164 165 total = len(index.layerBranches) 166 for i, id in enumerate(index.layerBranches): 167 # release as defined by toaster map to layerindex branch 168 release = li_branch_id_to_toaster_release[index.layerBranches[id].branch_id] 169 170 try: 171 lv, created = Layer_Version.objects.get_or_create( 172 layer=Layer.objects.get( 173 pk=li_layer_id_to_toaster_layer_id[index.layerBranches[id].layer_id]), 174 release=release 175 ) 176 except KeyError: 177 logger.warning( 178 "No such layerindex layer referenced by layerbranch %d" % 179 index.layerBranches[id].layer_id) 180 continue 181 182 if created: 183 lv.release = li_branch_id_to_toaster_release[index.layerBranches[id].branch_id] 184 lv.up_date = index.layerBranches[id].updated 185 lv.commit = index.layerBranches[id].actual_branch 186 lv.dirpath = index.layerBranches[id].vcs_subdir 187 lv.save() 188 189 li_layer_branch_id_to_toaster_lv_id[index.layerBranches[id].id] =\ 190 lv.pk 191 self.mini_progress("layer versions", i, total) 192 193 logger.info("Processing layer version dependencies") 194 195 dependlist = {} 196 for id in index.layerDependencies: 197 try: 198 lv = Layer_Version.objects.get( 199 pk=li_layer_branch_id_to_toaster_lv_id[index.layerDependencies[id].layerbranch_id]) 200 except Layer_Version.DoesNotExist as e: 201 continue 202 203 if lv not in dependlist: 204 dependlist[lv] = [] 205 try: 206 layer_id = li_layer_id_to_toaster_layer_id[index.layerDependencies[id].dependency_id] 207 208 dependlist[lv].append( 209 Layer_Version.objects.get(layer__pk=layer_id, 210 release=lv.release)) 211 212 except Layer_Version.DoesNotExist: 213 logger.warning("Cannot find layer version (ls:%s)," 214 "up_id:%s lv:%s" % 215 (self, index.layerDependencies[id].dependency_id, lv)) 216 217 total = len(dependlist) 218 for i, lv in enumerate(dependlist): 219 LayerVersionDependency.objects.filter(layer_version=lv).delete() 220 for lvd in dependlist[lv]: 221 LayerVersionDependency.objects.get_or_create(layer_version=lv, 222 depends_on=lvd) 223 self.mini_progress("Layer version dependencies", i, total) 224 225 # update Distros 226 logger.info("Processing distro information") 227 228 total = len(index.distros) 229 for i, id in enumerate(index.distros): 230 distro, created = Distro.objects.get_or_create( 231 name=index.distros[id].name, 232 layer_version=Layer_Version.objects.get( 233 pk=li_layer_branch_id_to_toaster_lv_id[index.distros[id].layerbranch_id])) 234 distro.up_date = index.distros[id].updated 235 distro.name = index.distros[id].name 236 distro.description = index.distros[id].description 237 distro.save() 238 self.mini_progress("distros", i, total) 239 240 # update machines 241 logger.info("Processing machine information") 242 243 total = len(index.machines) 244 for i, id in enumerate(index.machines): 245 mo, created = Machine.objects.get_or_create( 246 name=index.machines[id].name, 247 layer_version=Layer_Version.objects.get( 248 pk=li_layer_branch_id_to_toaster_lv_id[index.machines[id].layerbranch_id])) 249 mo.up_date = index.machines[id].updated 250 mo.name = index.machines[id].name 251 mo.description = index.machines[id].description 252 mo.save() 253 self.mini_progress("machines", i, total) 254 255 # update recipes; paginate by layer version / layer branch 256 logger.info("Processing recipe information") 257 258 total = len(index.recipes) 259 for i, id in enumerate(index.recipes): 260 try: 261 lv_id = li_layer_branch_id_to_toaster_lv_id[index.recipes[id].layerbranch_id] 262 lv = Layer_Version.objects.get(pk=lv_id) 263 264 ro, created = Recipe.objects.get_or_create( 265 layer_version=lv, 266 name=index.recipes[id].pn 267 ) 268 269 ro.layer_version = lv 270 ro.up_date = index.recipes[id].updated 271 ro.name = index.recipes[id].pn 272 ro.version = index.recipes[id].pv 273 ro.summary = index.recipes[id].summary 274 ro.description = index.recipes[id].description 275 ro.section = index.recipes[id].section 276 ro.license = index.recipes[id].license 277 ro.homepage = index.recipes[id].homepage 278 ro.bugtracker = index.recipes[id].bugtracker 279 ro.file_path = index.recipes[id].fullpath 280 ro.is_image = 'image' in index.recipes[id].inherits.split() 281 ro.save() 282 except Exception as e: 283 logger.warning("Failed saving recipe %s", e) 284 285 self.mini_progress("recipes", i, total) 286 287 os.system('setterm -cursor on') 288 289 def handle(self, **options): 290 self.update() 291