1#
2# BitBake Toaster Implementation
3#
4# Copyright (C) 2016-2017   Intel Corporation
5#
6# SPDX-License-Identifier: GPL-2.0-only
7#
8
9from django.core.management.base import BaseCommand
10
11from orm.models import Layer, Release, Layer_Version
12from orm.models import LayerVersionDependency, Machine, Recipe
13from orm.models import Distro
14from orm.models import ToasterSetting
15
16import os
17import sys
18
19import logging
20import threading
21import time
22logger = logging.getLogger("toaster")
23
24DEFAULT_LAYERINDEX_SERVER = "http://layers.openembedded.org/layerindex/api/"
25
26# Add path to bitbake modules for layerindexlib
27# lib/toaster/orm/management/commands/lsupdates.py (abspath)
28# lib/toaster/orm/management/commands (dirname)
29# lib/toaster/orm/management (dirname)
30# lib/toaster/orm (dirname)
31# lib/toaster/ (dirname)
32# lib/ (dirname)
33path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
34sys.path.insert(0, path)
35
36import layerindexlib
37
38
39class Spinner(threading.Thread):
40    """ A simple progress spinner to indicate download/parsing is happening"""
41    def __init__(self, *args, **kwargs):
42        super(Spinner, self).__init__(*args, **kwargs)
43        self.setDaemon(True)
44        self.signal = True
45
46    def run(self):
47        os.system('setterm -cursor off')
48        while self.signal:
49            for char in ["/", "-", "\\", "|"]:
50                sys.stdout.write("\r" + char)
51                sys.stdout.flush()
52                time.sleep(0.25)
53        os.system('setterm -cursor on')
54
55    def stop(self):
56        self.signal = False
57
58
59class Command(BaseCommand):
60    args = ""
61    help = "Updates locally cached information from a layerindex server"
62
63    def mini_progress(self, what, i, total):
64        i = i + 1
65        pec = (float(i)/float(total))*100
66
67        sys.stdout.write("\rUpdating %s %d%%" %
68                         (what,
69                          pec))
70        sys.stdout.flush()
71        if int(pec) == 100:
72            sys.stdout.write("\n")
73            sys.stdout.flush()
74
75    def update(self):
76        """
77            Fetches layer, recipe and machine information from a layerindex
78            server
79        """
80        os.system('setterm -cursor off')
81
82        self.apiurl = DEFAULT_LAYERINDEX_SERVER
83        if ToasterSetting.objects.filter(name='CUSTOM_LAYERINDEX_SERVER').count() == 1:
84            self.apiurl = ToasterSetting.objects.get(name = 'CUSTOM_LAYERINDEX_SERVER').value
85
86        assert self.apiurl is not None
87
88        # update branches; only those that we already have names listed in the
89        # Releases table
90        whitelist_branch_names = [rel.branch_name
91                                  for rel in Release.objects.all()]
92        if len(whitelist_branch_names) == 0:
93            raise Exception("Failed to make list of branches to fetch")
94
95        logger.info("Fetching metadata for %s",
96                    " ".join(whitelist_branch_names))
97
98        # We require a non-empty bb.data, but we can fake it with a dictionary
99        layerindex = layerindexlib.LayerIndex({"DUMMY" : "VALUE"})
100
101        http_progress = Spinner()
102        http_progress.start()
103
104        if whitelist_branch_names:
105            url_branches = ";branch=%s" % ','.join(whitelist_branch_names)
106        else:
107            url_branches = ""
108        layerindex.load_layerindex("%s%s" % (self.apiurl, url_branches))
109
110        http_progress.stop()
111
112        # We know we're only processing one entry, so we reference it here
113        # (this is cheating...)
114        index = layerindex.indexes[0]
115
116        # Map the layer index branches to toaster releases
117        li_branch_id_to_toaster_release = {}
118
119        logger.info("Processing releases")
120
121        total = len(index.branches)
122        for i, id in enumerate(index.branches):
123            li_branch_id_to_toaster_release[id] = \
124                    Release.objects.get(name=index.branches[id].name)
125            self.mini_progress("Releases", i, total)
126
127        # keep a track of the layerindex (li) id mappings so that
128        # layer_versions can be created for these layers later on
129        li_layer_id_to_toaster_layer_id = {}
130
131        logger.info("Processing layers")
132
133        total = len(index.layerItems)
134        for i, id in enumerate(index.layerItems):
135            try:
136                l, created = Layer.objects.get_or_create(name=index.layerItems[id].name)
137                l.up_date = index.layerItems[id].updated
138                l.summary = index.layerItems[id].summary
139                l.description = index.layerItems[id].description
140
141                if created:
142                    # predefined layers in the fixtures (for example poky.xml)
143                    # always preempt the Layer Index for these values
144                    l.vcs_url = index.layerItems[id].vcs_url
145                    l.vcs_web_url = index.layerItems[id].vcs_web_url
146                    l.vcs_web_tree_base_url = index.layerItems[id].vcs_web_tree_base_url
147                    l.vcs_web_file_base_url = index.layerItems[id].vcs_web_file_base_url
148                l.save()
149            except Layer.MultipleObjectsReturned:
150                logger.info("Skipped %s as we found multiple layers and "
151                            "don't know which to update" %
152                            index.layerItems[id].name)
153
154            li_layer_id_to_toaster_layer_id[id] = l.pk
155
156            self.mini_progress("layers", i, total)
157
158        # update layer_versions
159        logger.info("Processing layer versions")
160
161        # Map Layer index layer_branch object id to
162        # layer_version toaster object id
163        li_layer_branch_id_to_toaster_lv_id = {}
164
165        total = len(index.layerBranches)
166        for i, id in enumerate(index.layerBranches):
167            # release as defined by toaster map to layerindex branch
168            release = li_branch_id_to_toaster_release[index.layerBranches[id].branch_id]
169
170            try:
171                lv, created = Layer_Version.objects.get_or_create(
172                    layer=Layer.objects.get(
173                        pk=li_layer_id_to_toaster_layer_id[index.layerBranches[id].layer_id]),
174                    release=release
175                )
176            except KeyError:
177                logger.warning(
178                    "No such layerindex layer referenced by layerbranch %d" %
179                    index.layerBranches[id].layer_id)
180                continue
181
182            if created:
183                lv.release = li_branch_id_to_toaster_release[index.layerBranches[id].branch_id]
184                lv.up_date = index.layerBranches[id].updated
185                lv.commit = index.layerBranches[id].actual_branch
186                lv.dirpath = index.layerBranches[id].vcs_subdir
187                lv.save()
188
189            li_layer_branch_id_to_toaster_lv_id[index.layerBranches[id].id] =\
190                lv.pk
191            self.mini_progress("layer versions", i, total)
192
193        logger.info("Processing layer version dependencies")
194
195        dependlist = {}
196        for id in index.layerDependencies:
197            try:
198                lv = Layer_Version.objects.get(
199                    pk=li_layer_branch_id_to_toaster_lv_id[index.layerDependencies[id].layerbranch_id])
200            except Layer_Version.DoesNotExist as e:
201                continue
202
203            if lv not in dependlist:
204                dependlist[lv] = []
205            try:
206                layer_id = li_layer_id_to_toaster_layer_id[index.layerDependencies[id].dependency_id]
207
208                dependlist[lv].append(
209                    Layer_Version.objects.get(layer__pk=layer_id,
210                                              release=lv.release))
211
212            except Layer_Version.DoesNotExist:
213                logger.warning("Cannot find layer version (ls:%s),"
214                               "up_id:%s lv:%s" %
215                               (self, index.layerDependencies[id].dependency_id, lv))
216
217        total = len(dependlist)
218        for i, lv in enumerate(dependlist):
219            LayerVersionDependency.objects.filter(layer_version=lv).delete()
220            for lvd in dependlist[lv]:
221                LayerVersionDependency.objects.get_or_create(layer_version=lv,
222                                                             depends_on=lvd)
223            self.mini_progress("Layer version dependencies", i, total)
224
225        # update Distros
226        logger.info("Processing distro information")
227
228        total = len(index.distros)
229        for i, id in enumerate(index.distros):
230            distro, created = Distro.objects.get_or_create(
231                name=index.distros[id].name,
232                layer_version=Layer_Version.objects.get(
233                    pk=li_layer_branch_id_to_toaster_lv_id[index.distros[id].layerbranch_id]))
234            distro.up_date = index.distros[id].updated
235            distro.name = index.distros[id].name
236            distro.description = index.distros[id].description
237            distro.save()
238            self.mini_progress("distros", i, total)
239
240        # update machines
241        logger.info("Processing machine information")
242
243        total = len(index.machines)
244        for i, id in enumerate(index.machines):
245            mo, created = Machine.objects.get_or_create(
246                name=index.machines[id].name,
247                layer_version=Layer_Version.objects.get(
248                    pk=li_layer_branch_id_to_toaster_lv_id[index.machines[id].layerbranch_id]))
249            mo.up_date = index.machines[id].updated
250            mo.name = index.machines[id].name
251            mo.description = index.machines[id].description
252            mo.save()
253            self.mini_progress("machines", i, total)
254
255        # update recipes; paginate by layer version / layer branch
256        logger.info("Processing recipe information")
257
258        total = len(index.recipes)
259        for i, id in enumerate(index.recipes):
260            try:
261                lv_id = li_layer_branch_id_to_toaster_lv_id[index.recipes[id].layerbranch_id]
262                lv = Layer_Version.objects.get(pk=lv_id)
263
264                ro, created = Recipe.objects.get_or_create(
265                    layer_version=lv,
266                    name=index.recipes[id].pn
267                )
268
269                ro.layer_version = lv
270                ro.up_date = index.recipes[id].updated
271                ro.name = index.recipes[id].pn
272                ro.version = index.recipes[id].pv
273                ro.summary = index.recipes[id].summary
274                ro.description = index.recipes[id].description
275                ro.section = index.recipes[id].section
276                ro.license = index.recipes[id].license
277                ro.homepage = index.recipes[id].homepage
278                ro.bugtracker = index.recipes[id].bugtracker
279                ro.file_path = index.recipes[id].fullpath
280                ro.is_image = 'image' in index.recipes[id].inherits.split()
281                ro.save()
282            except Exception as e:
283                logger.warning("Failed saving recipe %s", e)
284
285            self.mini_progress("recipes", i, total)
286
287        os.system('setterm -cursor on')
288
289    def handle(self, **options):
290        self.update()
291