1#
2# BitBake Toaster Implementation
3#
4# Copyright (C) 2016        Intel Corporation
5#
6# SPDX-License-Identifier: GPL-2.0-only
7#
8# Please run flake8 on this file before sending patches
9
10import os
11import re
12import logging
13import json
14import subprocess
15from collections import Counter
16
17from orm.models import Project, ProjectTarget, Build, Layer_Version
18from orm.models import LayerVersionDependency, LayerSource, ProjectLayer
19from orm.models import Recipe, CustomImageRecipe, CustomImagePackage
20from orm.models import Layer, Target, Package, Package_Dependency
21from orm.models import ProjectVariable
22from bldcontrol.models import BuildRequest, BuildEnvironment
23from bldcontrol import bbcontroller
24
25from django.http import HttpResponse, JsonResponse
26from django.views.generic import View
27from django.urls import reverse
28from django.db.models import Q, F
29from django.db import Error
30from toastergui.templatetags.projecttags import filtered_filesizeformat
31
32# development/debugging support
33verbose = 2
34def _log(msg):
35    if 1 == verbose:
36        print(msg)
37    elif 2 == verbose:
38        f1=open('/tmp/toaster.log', 'a')
39        f1.write("|" + msg + "|\n" )
40        f1.close()
41
42logger = logging.getLogger("toaster")
43
44
45def error_response(error):
46    return JsonResponse({"error": error})
47
48
49class XhrBuildRequest(View):
50
51    def get(self, request, *args, **kwargs):
52        return HttpResponse()
53
54    @staticmethod
55    def cancel_build(br):
56        """Cancel a build request"""
57        try:
58            bbctrl = bbcontroller.BitbakeController(br.environment)
59            bbctrl.forceShutDown()
60        except:
61            # We catch a bunch of exceptions here because
62            # this is where the server has not had time to start up
63            # and the build request or build is in transit between
64            # processes.
65            # We can safely just set the build as cancelled
66            # already as it never got started
67            build = br.build
68            build.outcome = Build.CANCELLED
69            build.save()
70
71        # We now hand over to the buildinfohelper to update the
72        # build state once we've finished cancelling
73        br.state = BuildRequest.REQ_CANCELLING
74        br.save()
75
76    def post(self, request, *args, **kwargs):
77        """
78          Build control
79
80          Entry point: /xhr_buildrequest/<project_id>
81          Method: POST
82
83          Args:
84              id: id of build to change
85              buildCancel = build_request_id ...
86              buildDelete = id ...
87              targets = recipe_name ...
88
89          Returns:
90              {"error": "ok"}
91            or
92              {"error": <error message>}
93        """
94
95        project = Project.objects.get(pk=kwargs['pid'])
96
97        if 'buildCancel' in request.POST:
98            for i in request.POST['buildCancel'].strip().split(" "):
99                try:
100                    br = BuildRequest.objects.get(project=project, pk=i)
101                    self.cancel_build(br)
102                except BuildRequest.DoesNotExist:
103                    return error_response('No such build request id %s' % i)
104
105            return error_response('ok')
106
107        if 'buildDelete' in request.POST:
108            for i in request.POST['buildDelete'].strip().split(" "):
109                try:
110                    BuildRequest.objects.select_for_update().get(
111                        project=project,
112                        pk=i,
113                        state__lte=BuildRequest.REQ_DELETED).delete()
114
115                except BuildRequest.DoesNotExist:
116                    pass
117            return error_response("ok")
118
119        if 'targets' in request.POST:
120            ProjectTarget.objects.filter(project=project).delete()
121            s = str(request.POST['targets'])
122            for t in re.sub(r'[;%|"]', '', s).split(" "):
123                if ":" in t:
124                    target, task = t.split(":")
125                else:
126                    target = t
127                    task = ""
128                ProjectTarget.objects.create(project=project,
129                                             target=target,
130                                             task=task)
131            project.schedule_build()
132
133            return error_response('ok')
134
135        response = HttpResponse()
136        response.status_code = 500
137        return response
138
139
140class XhrProjectUpdate(View):
141
142    def get(self, request, *args, **kwargs):
143        return HttpResponse()
144
145    def post(self, request, *args, **kwargs):
146        """
147          Project Update
148
149          Entry point: /xhr_projectupdate/<project_id>
150          Method: POST
151
152          Args:
153              pid: pid of project to update
154
155          Returns:
156              {"error": "ok"}
157            or
158              {"error": <error message>}
159        """
160
161        project = Project.objects.get(pk=kwargs['pid'])
162        logger.debug("ProjectUpdateCallback:project.pk=%d,project.builddir=%s" % (project.pk,project.builddir))
163
164        if 'do_update' in request.POST:
165
166            # Extract any default image recipe
167            if 'default_image' in request.POST:
168                project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,str(request.POST['default_image']))
169            else:
170                project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,'')
171
172            logger.debug("ProjectUpdateCallback:Chain to the build request")
173
174            # Chain to the build request
175            xhrBuildRequest = XhrBuildRequest()
176            return xhrBuildRequest.post(request, *args, **kwargs)
177
178        logger.warning("ERROR:XhrProjectUpdate")
179        response = HttpResponse()
180        response.status_code = 500
181        return response
182
183class XhrSetDefaultImageUrl(View):
184
185    def get(self, request, *args, **kwargs):
186        return HttpResponse()
187
188    def post(self, request, *args, **kwargs):
189        """
190          Project Update
191
192          Entry point: /xhr_setdefaultimage/<project_id>
193          Method: POST
194
195          Args:
196              pid: pid of project to update default image
197
198          Returns:
199              {"error": "ok"}
200            or
201              {"error": <error message>}
202        """
203
204        project = Project.objects.get(pk=kwargs['pid'])
205        logger.debug("XhrSetDefaultImageUrl:project.pk=%d" % (project.pk))
206
207        # set any default image recipe
208        if 'targets' in request.POST:
209            default_target = str(request.POST['targets'])
210            project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,default_target)
211            logger.debug("XhrSetDefaultImageUrl,project.pk=%d,project.builddir=%s" % (project.pk,project.builddir))
212            return error_response('ok')
213
214        logger.warning("ERROR:XhrSetDefaultImageUrl")
215        response = HttpResponse()
216        response.status_code = 500
217        return response
218
219
220#
221# Layer Management
222#
223# Rules for 'local_source_dir' layers
224#  * Layers must have a unique name in the Layers table
225#  * A 'local_source_dir' layer is supposed to be shared
226#    by all projects that use it, so that it can have the
227#    same logical name
228#  * Each project that uses a layer will have its own
229#    LayerVersion and Project Layer for it
230#  * During the Paroject delete process, when the last
231#    LayerVersion for a 'local_source_dir' layer is deleted
232#    then the Layer record is deleted to remove orphans
233#
234
235def scan_layer_content(layer,layer_version):
236    # if this is a local layer directory, we can immediately scan its content
237    if layer.local_source_dir:
238        try:
239            # recipes-*/*/*.bb
240            cmd = '%s %s' % ('ls', os.path.join(layer.local_source_dir,'recipes-*/*/*.bb'))
241            recipes_list = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read()
242            recipes_list = recipes_list.decode("utf-8").strip()
243            if recipes_list and 'No such' not in recipes_list:
244                for recipe in recipes_list.split('\n'):
245                    recipe_path = recipe[recipe.rfind('recipes-'):]
246                    recipe_name = recipe[recipe.rfind('/')+1:].replace('.bb','')
247                    recipe_ver = recipe_name.rfind('_')
248                    if recipe_ver > 0:
249                        recipe_name = recipe_name[0:recipe_ver]
250                    if recipe_name:
251                        ro, created = Recipe.objects.get_or_create(
252                            layer_version=layer_version,
253                            name=recipe_name
254                        )
255                        if created:
256                            ro.file_path = recipe_path
257                            ro.summary = 'Recipe %s from layer %s' % (recipe_name,layer.name)
258                            ro.description = ro.summary
259                        ro.save()
260
261        except Exception as e:
262            logger.warning("ERROR:scan_layer_content: %s" % e)
263
264class XhrLayer(View):
265    """ Delete, Get, Add and Update Layer information
266
267        Methods: GET POST DELETE PUT
268    """
269
270    def get(self, request, *args, **kwargs):
271        """
272        Get layer information
273
274        Method: GET
275        Entry point: /xhr_layer/<project id>/<layerversion_id>
276        """
277
278        try:
279            layer_version = Layer_Version.objects.get(
280                pk=kwargs['layerversion_id'])
281
282            project = Project.objects.get(pk=kwargs['pid'])
283
284            project_layers = ProjectLayer.objects.filter(
285                project=project).values_list("layercommit_id",
286                                             flat=True)
287
288            ret = {
289                'error': 'ok',
290                'id': layer_version.pk,
291                'name': layer_version.layer.name,
292                'layerdetailurl':
293                layer_version.get_detailspage_url(project.pk),
294                'vcs_ref': layer_version.get_vcs_reference(),
295                'vcs_url': layer_version.layer.vcs_url,
296                'local_source_dir': layer_version.layer.local_source_dir,
297                'layerdeps': {
298                    "list": [
299                        {
300                            "id": dep.id,
301                            "name": dep.layer.name,
302                            "layerdetailurl":
303                            dep.get_detailspage_url(project.pk),
304                            "vcs_url": dep.layer.vcs_url,
305                            "vcs_reference": dep.get_vcs_reference()
306                        }
307                        for dep in layer_version.get_alldeps(project.id)]
308                },
309                'projectlayers': list(project_layers)
310            }
311
312            return JsonResponse(ret)
313        except Layer_Version.DoesNotExist:
314            error_response("No such layer")
315
316    def post(self, request, *args, **kwargs):
317        """
318          Update a layer
319
320          Method: POST
321          Entry point: /xhr_layer/<layerversion_id>
322
323          Args:
324              vcs_url, dirpath, commit, up_branch, summary, description,
325              local_source_dir
326
327              add_dep = append a layerversion_id as a dependency
328              rm_dep = remove a layerversion_id as a depedency
329          Returns:
330              {"error": "ok"}
331            or
332              {"error": <error message>}
333        """
334
335        try:
336            # We currently only allow Imported layers to be edited
337            layer_version = Layer_Version.objects.get(
338                id=kwargs['layerversion_id'],
339                project=kwargs['pid'],
340                layer_source=LayerSource.TYPE_IMPORTED)
341
342        except Layer_Version.DoesNotExist:
343            return error_response("Cannot find imported layer to update")
344
345        if "vcs_url" in request.POST:
346            layer_version.layer.vcs_url = request.POST["vcs_url"]
347        if "dirpath" in request.POST:
348            layer_version.dirpath = request.POST["dirpath"]
349        if "commit" in request.POST:
350            layer_version.commit = request.POST["commit"]
351            layer_version.branch = request.POST["commit"]
352        if "summary" in request.POST:
353            layer_version.layer.summary = request.POST["summary"]
354        if "description" in request.POST:
355            layer_version.layer.description = request.POST["description"]
356        if "local_source_dir" in request.POST:
357            layer_version.layer.local_source_dir = \
358                request.POST["local_source_dir"]
359
360        if "add_dep" in request.POST:
361            lvd = LayerVersionDependency(
362                layer_version=layer_version,
363                depends_on_id=request.POST["add_dep"])
364            lvd.save()
365
366        if "rm_dep" in request.POST:
367            rm_dep = LayerVersionDependency.objects.get(
368                layer_version=layer_version,
369                depends_on_id=request.POST["rm_dep"])
370            rm_dep.delete()
371
372        try:
373            layer_version.layer.save()
374            layer_version.save()
375        except Exception as e:
376            return error_response("Could not update layer version entry: %s"
377                                  % e)
378
379        return error_response("ok")
380
381    def put(self, request, *args, **kwargs):
382        """ Add a new layer
383
384        Method: PUT
385        Entry point: /xhr_layer/<project id>/
386        Args:
387            project_id, name,
388            [vcs_url, dir_path, git_ref], [local_source_dir], [layer_deps
389            (csv)]
390
391        """
392
393        try:
394            project = Project.objects.get(pk=kwargs['pid'])
395
396            layer_data = json.loads(request.body.decode('utf-8'))
397
398            # We require a unique layer name as otherwise the lists of layers
399            # becomes very confusing
400            existing_layers = \
401                project.get_all_compatible_layer_versions().values_list(
402                    "layer__name",
403                    flat=True)
404
405            add_to_project = False
406            layer_deps_added = []
407            if 'add_to_project' in layer_data:
408                add_to_project = True
409
410            if layer_data['name'] in existing_layers:
411                return JsonResponse({"error": "layer-name-exists"})
412
413            if ('local_source_dir' in layer_data):
414                # Local layer can be shared across projects. They have no 'release'
415                # and are not included in get_all_compatible_layer_versions() above
416                layer,created = Layer.objects.get_or_create(name=layer_data['name'])
417                _log("Local Layer created=%s" % created)
418            else:
419                layer = Layer.objects.create(name=layer_data['name'])
420
421            layer_version = Layer_Version.objects.create(
422                layer=layer,
423                project=project,
424                layer_source=LayerSource.TYPE_IMPORTED)
425
426            # Local layer
427            if ('local_source_dir' in layer_data): ### and layer.local_source_dir:
428                layer.local_source_dir = layer_data['local_source_dir']
429            # git layer
430            elif 'vcs_url' in layer_data:
431                layer.vcs_url = layer_data['vcs_url']
432                layer_version.dirpath = layer_data['dir_path']
433                layer_version.commit = layer_data['git_ref']
434                layer_version.branch = layer_data['git_ref']
435
436            layer.save()
437            layer_version.save()
438
439            if add_to_project:
440                ProjectLayer.objects.get_or_create(
441                    layercommit=layer_version, project=project)
442
443            # Add the layer dependencies
444            if 'layer_deps' in layer_data:
445                for layer_dep_id in layer_data['layer_deps'].split(","):
446                    layer_dep = Layer_Version.objects.get(pk=layer_dep_id)
447                    LayerVersionDependency.objects.get_or_create(
448                        layer_version=layer_version, depends_on=layer_dep)
449
450                    # Add layer deps to the project if specified
451                    if add_to_project:
452                        created, pl = ProjectLayer.objects.get_or_create(
453                            layercommit=layer_dep, project=project)
454                        layer_deps_added.append(
455                            {'name': layer_dep.layer.name,
456                             'layerdetailurl':
457                             layer_dep.get_detailspage_url(project.pk)})
458
459            # Scan the layer's content and update components
460            scan_layer_content(layer,layer_version)
461
462        except Layer_Version.DoesNotExist:
463            return error_response("layer-dep-not-found")
464        except Project.DoesNotExist:
465            return error_response("project-not-found")
466        except KeyError:
467            return error_response("incorrect-parameters")
468
469        return JsonResponse({'error': "ok",
470                             'imported_layer': {
471                                 'name': layer.name,
472                                 'layerdetailurl':
473                                 layer_version.get_detailspage_url()},
474                             'deps_added': layer_deps_added})
475
476    def delete(self, request, *args, **kwargs):
477        """ Delete an imported layer
478
479        Method: DELETE
480        Entry point: /xhr_layer/<projed id>/<layerversion_id>
481
482        """
483        try:
484            # We currently only allow Imported layers to be deleted
485            layer_version = Layer_Version.objects.get(
486                id=kwargs['layerversion_id'],
487                project=kwargs['pid'],
488                layer_source=LayerSource.TYPE_IMPORTED)
489        except Layer_Version.DoesNotExist:
490            return error_response("Cannot find imported layer to delete")
491
492        try:
493            ProjectLayer.objects.get(project=kwargs['pid'],
494                                     layercommit=layer_version).delete()
495        except ProjectLayer.DoesNotExist:
496            pass
497
498        layer_version.layer.delete()
499        layer_version.delete()
500
501        return JsonResponse({
502            "error": "ok",
503            "gotoUrl": reverse('projectlayers', args=(kwargs['pid'],))
504        })
505
506
507class XhrCustomRecipe(View):
508    """ Create a custom image recipe """
509
510    def post(self, request, *args, **kwargs):
511        """
512        Custom image recipe REST API
513
514        Entry point: /xhr_customrecipe/
515        Method: POST
516
517        Args:
518            name: name of custom recipe to create
519            project: target project id of orm.models.Project
520            base: base recipe id of orm.models.Recipe
521
522        Returns:
523            {"error": "ok",
524             "url": <url of the created recipe>}
525            or
526            {"error": <error message>}
527        """
528        # check if request has all required parameters
529        for param in ('name', 'project', 'base'):
530            if param not in request.POST:
531                return error_response("Missing parameter '%s'" % param)
532
533        # get project and baserecipe objects
534        params = {}
535        for name, model in [("project", Project),
536                            ("base", Recipe)]:
537            value = request.POST[name]
538            try:
539                params[name] = model.objects.get(id=value)
540            except model.DoesNotExist:
541                return error_response("Invalid %s id %s" % (name, value))
542
543        # create custom recipe
544        try:
545
546            # Only allowed chars in name are a-z, 0-9 and -
547            if re.search(r'[^a-z|0-9|-]', request.POST["name"]):
548                return error_response("invalid-name")
549
550            custom_images = CustomImageRecipe.objects.all()
551
552            # Are there any recipes with this name already in our project?
553            existing_image_recipes_in_project = custom_images.filter(
554                name=request.POST["name"], project=params["project"])
555
556            if existing_image_recipes_in_project.count() > 0:
557                return error_response("image-already-exists")
558
559            # Are there any recipes with this name which aren't custom
560            # image recipes?
561            custom_image_ids = custom_images.values_list('id', flat=True)
562            existing_non_image_recipes = Recipe.objects.filter(
563                Q(name=request.POST["name"]) & ~Q(pk__in=custom_image_ids)
564            )
565
566            if existing_non_image_recipes.count() > 0:
567                return error_response("recipe-already-exists")
568
569            # create layer 'Custom layer' and verion if needed
570            layer, l_created = Layer.objects.get_or_create(
571                name=CustomImageRecipe.LAYER_NAME,
572                summary="Layer for custom recipes")
573
574            if l_created:
575                layer.local_source_dir = "toaster_created_layer"
576                layer.save()
577
578            # Check if we have a layer version already
579            # We don't use get_or_create here because the dirpath will change
580            # and is a required field
581            lver = Layer_Version.objects.filter(Q(project=params['project']) &
582                                                Q(layer=layer) &
583                                                Q(build=None)).last()
584            if lver is None:
585                lver, lv_created = Layer_Version.objects.get_or_create(
586                    project=params['project'],
587                    layer=layer,
588                    layer_source=LayerSource.TYPE_LOCAL,
589                    dirpath="toaster_created_layer")
590
591            # Add a dependency on our layer to the base recipe's layer
592            LayerVersionDependency.objects.get_or_create(
593                layer_version=lver,
594                depends_on=params["base"].layer_version)
595
596            # Add it to our current project if needed
597            ProjectLayer.objects.get_or_create(project=params['project'],
598                                               layercommit=lver,
599                                               optional=False)
600
601            # Create the actual recipe
602            recipe, r_created = CustomImageRecipe.objects.get_or_create(
603                name=request.POST["name"],
604                base_recipe=params["base"],
605                project=params["project"],
606                layer_version=lver,
607                is_image=True)
608
609            # If we created the object then setup these fields. They may get
610            # overwritten later on and cause the get_or_create to create a
611            # duplicate if they've changed.
612            if r_created:
613                recipe.file_path = request.POST["name"]
614                recipe.license = "MIT"
615                recipe.version = "0.1"
616                recipe.save()
617
618        except Error as err:
619            return error_response("Can't create custom recipe: %s" % err)
620
621        # Find the package list from the last build of this recipe/target
622        target = Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) &
623                                       Q(build__project=params['project']) &
624                                       (Q(target=params['base'].name) |
625                                        Q(target=recipe.name))).last()
626        if target:
627            # Copy in every package
628            # We don't want these packages to be linked to anything because
629            # that underlying data may change e.g. delete a build
630            for tpackage in target.target_installed_package_set.all():
631                try:
632                    built_package = tpackage.package
633                    # The package had no recipe information so is a ghost
634                    # package skip it
635                    if built_package.recipe is None:
636                        continue
637
638                    config_package = CustomImagePackage.objects.get(
639                        name=built_package.name)
640
641                    recipe.includes_set.add(config_package)
642                except Exception as e:
643                    logger.warning("Error adding package %s %s" %
644                                   (tpackage.package.name, e))
645                    pass
646
647        # pre-create layer directory structure, so that other builds
648        # are not blocked by this new recipe dependecy
649        # NOTE: this is parallel code to 'localhostbecontroller.py'
650        be = BuildEnvironment.objects.all()[0]
651        layerpath = os.path.join(be.builddir,
652                                 CustomImageRecipe.LAYER_NAME)
653        for name in ("conf", "recipes"):
654            path = os.path.join(layerpath, name)
655            if not os.path.isdir(path):
656                os.makedirs(path)
657        # pre-create layer.conf
658        config = os.path.join(layerpath, "conf", "layer.conf")
659        if not os.path.isfile(config):
660            with open(config, "w") as conf:
661                conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n')
662        # pre-create new image's recipe file
663        recipe_path = os.path.join(layerpath, "recipes", "%s.bb" %
664                                   recipe.name)
665        with open(recipe_path, "w") as recipef:
666            content = recipe.generate_recipe_file_contents()
667            if not content:
668                # Delete this incomplete image recipe object
669                recipe.delete()
670                return error_response("recipe-parent-not-exist")
671            else:
672                recipef.write(recipe.generate_recipe_file_contents())
673
674        return JsonResponse(
675            {"error": "ok",
676             "packages": recipe.get_all_packages().count(),
677             "url": reverse('customrecipe', args=(params['project'].pk,
678                                                  recipe.id))})
679
680
681class XhrCustomRecipeId(View):
682    """
683    Set of ReST API processors working with recipe id.
684
685    Entry point: /xhr_customrecipe/<recipe_id>
686
687    Methods:
688        GET - Get details of custom image recipe
689        DELETE - Delete custom image recipe
690
691    Returns:
692        GET:
693        {"error": "ok",
694        "info": dictionary of field name -> value pairs
695        of the CustomImageRecipe model}
696        DELETE:
697        {"error": "ok"}
698          or
699        {"error": <error message>}
700    """
701    @staticmethod
702    def _get_ci_recipe(recipe_id):
703        """ Get Custom Image recipe or return an error response"""
704        try:
705            custom_recipe = \
706                CustomImageRecipe.objects.get(pk=recipe_id)
707            return custom_recipe, None
708
709        except CustomImageRecipe.DoesNotExist:
710            return None, error_response("Custom recipe with id=%s "
711                                        "not found" % recipe_id)
712
713    def get(self, request, *args, **kwargs):
714        custom_recipe, error = self._get_ci_recipe(kwargs['recipe_id'])
715        if error:
716            return error
717
718        if request.method == 'GET':
719            info = {"id": custom_recipe.id,
720                    "name": custom_recipe.name,
721                    "base_recipe_id": custom_recipe.base_recipe.id,
722                    "project_id": custom_recipe.project.id}
723
724            return JsonResponse({"error": "ok", "info": info})
725
726    def delete(self, request, *args, **kwargs):
727        custom_recipe, error = self._get_ci_recipe(kwargs['recipe_id'])
728        if error:
729            return error
730
731        project = custom_recipe.project
732
733        custom_recipe.delete()
734        return JsonResponse({"error": "ok",
735                             "gotoUrl": reverse("projectcustomimages",
736                                                args=(project.pk,))})
737
738
739class XhrCustomRecipePackages(View):
740    """
741    ReST API to add/remove packages to/from custom recipe.
742
743    Entry point: /xhr_customrecipe/<recipe_id>/packages/<package_id>
744    Methods:
745         PUT - Add package to the recipe
746         DELETE - Delete package from the recipe
747         GET - Get package information
748
749     Returns:
750         {"error": "ok"}
751          or
752          {"error": <error message>}
753    """
754    @staticmethod
755    def _get_package(package_id):
756        try:
757            package = CustomImagePackage.objects.get(pk=package_id)
758            return package, None
759        except Package.DoesNotExist:
760            return None, error_response("Package with id=%s "
761                                        "not found" % package_id)
762
763    def _traverse_dependents(self, next_package_id,
764                             rev_deps, all_current_packages, tree_level=0):
765        """
766        Recurse through reverse dependency tree for next_package_id.
767        Limit the reverse dependency search to packages not already scanned,
768        that is, not already in rev_deps.
769        Limit the scan to a depth (tree_level) not exceeding the count of
770        all packages in the custom image, and if that depth is exceeded
771        return False, pop out of the recursion, and write a warning
772        to the log, but this is unlikely, suggesting a dependency loop
773        not caught by bitbake.
774        On return, the input/output arg rev_deps is appended with queryset
775        dictionary elements, annotated for use in the customimage template.
776        The list has unsorted, but unique elements.
777        """
778        max_dependency_tree_depth = all_current_packages.count()
779        if tree_level >= max_dependency_tree_depth:
780            logger.warning(
781                "The number of reverse dependencies "
782                "for this package exceeds " + max_dependency_tree_depth +
783                " and the remaining reverse dependencies will not be removed")
784            return True
785
786        package = CustomImagePackage.objects.get(id=next_package_id)
787        dependents = \
788            package.package_dependencies_target.annotate(
789                name=F('package__name'),
790                pk=F('package__pk'),
791                size=F('package__size'),
792            ).values("name", "pk", "size").exclude(
793                ~Q(pk__in=all_current_packages)
794            )
795
796        for pkg in dependents:
797            if pkg in rev_deps:
798                # already seen, skip dependent search
799                continue
800
801            rev_deps.append(pkg)
802            if (self._traverse_dependents(pkg["pk"], rev_deps,
803                                          all_current_packages,
804                                          tree_level+1)):
805                return True
806
807        return False
808
809    def _get_all_dependents(self, package_id, all_current_packages):
810        """
811        Returns sorted list of recursive reverse dependencies for package_id,
812        as a list of dictionary items, by recursing through dependency
813        relationships.
814        """
815        rev_deps = []
816        self._traverse_dependents(package_id, rev_deps, all_current_packages)
817        rev_deps = sorted(rev_deps, key=lambda x: x["name"])
818        return rev_deps
819
820    def get(self, request, *args, **kwargs):
821        recipe, error = XhrCustomRecipeId._get_ci_recipe(
822            kwargs['recipe_id'])
823        if error:
824            return error
825
826        # If no package_id then list all the current packages
827        if not kwargs['package_id']:
828            total_size = 0
829            packages = recipe.get_all_packages().values("id",
830                                                        "name",
831                                                        "version",
832                                                        "size")
833            for package in packages:
834                package['size_formatted'] = \
835                    filtered_filesizeformat(package['size'])
836                total_size += package['size']
837
838            return JsonResponse({"error": "ok",
839                                 "packages": list(packages),
840                                 "total": len(packages),
841                                 "total_size": total_size,
842                                 "total_size_formatted":
843                                 filtered_filesizeformat(total_size)})
844        else:
845            package, error = XhrCustomRecipePackages._get_package(
846                kwargs['package_id'])
847            if error:
848                return error
849
850            all_current_packages = recipe.get_all_packages()
851
852            # Dependencies for package which aren't satisfied by the
853            # current packages in the custom image recipe
854            deps = package.package_dependencies_source.for_target_or_none(
855                recipe.name)['packages'].annotate(
856                name=F('depends_on__name'),
857                pk=F('depends_on__pk'),
858                size=F('depends_on__size'),
859                ).values("name", "pk", "size").filter(
860                # There are two depends types we don't know why
861                (Q(dep_type=Package_Dependency.TYPE_TRDEPENDS) |
862                 Q(dep_type=Package_Dependency.TYPE_RDEPENDS)) &
863                ~Q(pk__in=all_current_packages)
864                )
865
866            # Reverse dependencies which are needed by packages that are
867            # in the image. Recursive search providing all dependents,
868            # not just immediate dependents.
869            reverse_deps = self._get_all_dependents(kwargs['package_id'],
870                                                    all_current_packages)
871            total_size_deps = 0
872            total_size_reverse_deps = 0
873
874            for dep in deps:
875                dep['size_formatted'] = \
876                    filtered_filesizeformat(dep['size'])
877                total_size_deps += dep['size']
878
879            for dep in reverse_deps:
880                dep['size_formatted'] = \
881                    filtered_filesizeformat(dep['size'])
882                total_size_reverse_deps += dep['size']
883
884            return JsonResponse(
885                {"error": "ok",
886                 "id": package.pk,
887                 "name": package.name,
888                 "version": package.version,
889                 "unsatisfied_dependencies": list(deps),
890                 "unsatisfied_dependencies_size": total_size_deps,
891                 "unsatisfied_dependencies_size_formatted":
892                 filtered_filesizeformat(total_size_deps),
893                 "reverse_dependencies": list(reverse_deps),
894                 "reverse_dependencies_size": total_size_reverse_deps,
895                 "reverse_dependencies_size_formatted":
896                 filtered_filesizeformat(total_size_reverse_deps)})
897
898    def put(self, request, *args, **kwargs):
899        recipe, error = XhrCustomRecipeId._get_ci_recipe(kwargs['recipe_id'])
900        package, error = self._get_package(kwargs['package_id'])
901        if error:
902            return error
903
904        included_packages = recipe.includes_set.values_list('pk',
905                                                            flat=True)
906
907        # If we're adding back a package which used to be included in this
908        # image all we need to do is remove it from the excludes
909        if package.pk in included_packages:
910            try:
911                recipe.excludes_set.remove(package)
912                return {"error": "ok"}
913            except Package.DoesNotExist:
914                return error_response("Package %s not found in excludes"
915                                      " but was in included list" %
916                                      package.name)
917        else:
918            recipe.appends_set.add(package)
919            # Make sure that package is not in the excludes set
920            try:
921                recipe.excludes_set.remove(package)
922            except:
923                pass
924
925        # Add the dependencies we think will be added to the recipe
926        # as a result of appending this package.
927        # TODO this should recurse down the entire deps tree
928        for dep in package.package_dependencies_source.all_depends():
929            try:
930                cust_package = CustomImagePackage.objects.get(
931                    name=dep.depends_on.name)
932
933                recipe.includes_set.add(cust_package)
934                try:
935                    # When adding the pre-requisite package, make
936                    # sure it's not in the excluded list from a
937                    # prior removal.
938                    recipe.excludes_set.remove(cust_package)
939                except package.DoesNotExist:
940                    # Don't care if the package had never been excluded
941                    pass
942            except:
943                logger.warning("Could not add package's suggested"
944                               "dependencies to the list")
945        return JsonResponse({"error": "ok"})
946
947    def delete(self, request, *args, **kwargs):
948        recipe, error = XhrCustomRecipeId._get_ci_recipe(kwargs['recipe_id'])
949        package, error = self._get_package(kwargs['package_id'])
950        if error:
951            return error
952
953        try:
954            included_packages = recipe.includes_set.values_list('pk',
955                                                                flat=True)
956            # If we're deleting a package which is included we need to
957            # Add it to the excludes list.
958            if package.pk in included_packages:
959                recipe.excludes_set.add(package)
960            else:
961                recipe.appends_set.remove(package)
962
963            # remove dependencies as well
964            all_current_packages = recipe.get_all_packages()
965
966            reverse_deps_dictlist = self._get_all_dependents(
967                package.pk,
968                all_current_packages)
969
970            ids = [entry['pk'] for entry in reverse_deps_dictlist]
971            reverse_deps = CustomImagePackage.objects.filter(id__in=ids)
972            for r in reverse_deps:
973                try:
974                    if r.id in included_packages:
975                        recipe.excludes_set.add(r)
976                    else:
977                        recipe.appends_set.remove(r)
978                except:
979                    pass
980
981            return JsonResponse({"error": "ok"})
982        except CustomImageRecipe.DoesNotExist:
983            return error_response("Tried to remove package that wasn't"
984                                  " present")
985
986
987class XhrProject(View):
988    """ Create, delete or edit a project
989
990    Entry point: /xhr_project/<project_id>
991    """
992    def post(self, request, *args, **kwargs):
993        """
994          Edit project control
995
996          Args:
997              layerAdd = layer_version_id layer_version_id ...
998              layerDel = layer_version_id layer_version_id ...
999              projectName = new_project_name
1000              machineName = new_machine_name
1001
1002          Returns:
1003              {"error": "ok"}
1004            or
1005              {"error": <error message>}
1006        """
1007        try:
1008            prj = Project.objects.get(pk=kwargs['project_id'])
1009        except Project.DoesNotExist:
1010            return error_response("No such project")
1011
1012        # Add layers
1013        if 'layerAdd' in request.POST and len(request.POST['layerAdd']) > 0:
1014            for layer_version_id in request.POST['layerAdd'].split(','):
1015                try:
1016                    lv = Layer_Version.objects.get(pk=int(layer_version_id))
1017                    ProjectLayer.objects.get_or_create(project=prj,
1018                                                       layercommit=lv)
1019                except Layer_Version.DoesNotExist:
1020                    return error_response("Layer version %s asked to add "
1021                                          "doesn't exist" % layer_version_id)
1022
1023        # Remove layers
1024        if 'layerDel' in request.POST and len(request.POST['layerDel']) > 0:
1025            layer_version_ids = request.POST['layerDel'].split(',')
1026            ProjectLayer.objects.filter(
1027                project=prj,
1028                layercommit_id__in=layer_version_ids).delete()
1029
1030        # Project name change
1031        if 'projectName' in request.POST:
1032            prj.name = request.POST['projectName']
1033            prj.save()
1034
1035        # Machine name change
1036        if 'machineName' in request.POST:
1037            machinevar = prj.projectvariable_set.get(name="MACHINE")
1038            machinevar.value = request.POST['machineName']
1039            machinevar.save()
1040
1041        # Distro name change
1042        if 'distroName' in request.POST:
1043            distrovar = prj.projectvariable_set.get(name="DISTRO")
1044            distrovar.value = request.POST['distroName']
1045            distrovar.save()
1046
1047        return JsonResponse({"error": "ok"})
1048
1049    def get(self, request, *args, **kwargs):
1050        """
1051        Returns:
1052            json object representing the current project
1053        or:
1054            {"error": <error message>}
1055        """
1056
1057        try:
1058            project = Project.objects.get(pk=kwargs['project_id'])
1059        except Project.DoesNotExist:
1060            return error_response("Project %s does not exist" %
1061                                  kwargs['project_id'])
1062
1063        # Create the frequently built targets list
1064
1065        freqtargets = Counter(Target.objects.filter(
1066            Q(build__project=project),
1067            ~Q(build__outcome=Build.IN_PROGRESS)
1068        ).order_by("target").values_list("target", flat=True))
1069
1070        freqtargets = freqtargets.most_common(5)
1071
1072        # We now have the targets in order of frequency but if there are two
1073        # with the same frequency then we need to make sure those are in
1074        # alphabetical order without losing the frequency ordering
1075
1076        tmp = []
1077        switch = None
1078        for i, freqtartget in enumerate(freqtargets):
1079            target, count = freqtartget
1080            try:
1081                target_next, count_next = freqtargets[i+1]
1082                if count == count_next and target > target_next:
1083                    switch = target
1084                    continue
1085            except IndexError:
1086                pass
1087
1088            tmp.append(target)
1089
1090            if switch:
1091                tmp.append(switch)
1092                switch = None
1093
1094        freqtargets = tmp
1095
1096        layers = []
1097        for layer in project.projectlayer_set.all():
1098            layers.append({
1099                "id": layer.layercommit.pk,
1100                "name": layer.layercommit.layer.name,
1101                "vcs_url": layer.layercommit.layer.vcs_url,
1102                "local_source_dir": layer.layercommit.layer.local_source_dir,
1103                "vcs_reference": layer.layercommit.get_vcs_reference(),
1104                "url": layer.layercommit.layer.layer_index_url,
1105                "layerdetailurl": layer.layercommit.get_detailspage_url(
1106                    project.pk),
1107                "xhrLayerUrl": reverse("xhr_layer",
1108                                       args=(project.pk,
1109                                             layer.layercommit.pk)),
1110                "layersource": layer.layercommit.layer_source
1111            })
1112
1113        data = {
1114            "name": project.name,
1115            "layers": layers,
1116            "freqtargets": freqtargets,
1117        }
1118
1119        if project.release is not None:
1120            data['release'] = {
1121                "id": project.release.pk,
1122                "name": project.release.name,
1123                "description": project.release.description
1124            }
1125
1126        try:
1127            data["machine"] = {"name":
1128                               project.projectvariable_set.get(
1129                                   name="MACHINE").value}
1130        except ProjectVariable.DoesNotExist:
1131            data["machine"] = None
1132        try:
1133            data["distro"] = {"name":
1134                               project.projectvariable_set.get(
1135                                   name="DISTRO").value}
1136        except ProjectVariable.DoesNotExist:
1137            data["distro"] = None
1138
1139        data['error'] = "ok"
1140
1141        return JsonResponse(data)
1142
1143    def put(self, request, *args, **kwargs):
1144        # TODO create new project api
1145        return HttpResponse()
1146
1147    def delete(self, request, *args, **kwargs):
1148        """Delete a project. Cancels any builds in progress"""
1149        try:
1150            project = Project.objects.get(pk=kwargs['project_id'])
1151            # Cancel any builds in progress
1152            for br in BuildRequest.objects.filter(
1153                    project=project,
1154                    state=BuildRequest.REQ_INPROGRESS):
1155                XhrBuildRequest.cancel_build(br)
1156
1157            # gather potential orphaned local layers attached to this project
1158            project_local_layer_list = []
1159            for pl in ProjectLayer.objects.filter(project=project):
1160                if pl.layercommit.layer_source == LayerSource.TYPE_IMPORTED:
1161                    project_local_layer_list.append(pl.layercommit.layer)
1162
1163            # deep delete the project and its dependencies
1164            project.delete()
1165
1166            # delete any local layers now orphaned
1167            _log("LAYER_ORPHAN_CHECK:Check for orphaned layers")
1168            for layer in project_local_layer_list:
1169                layer_refs = Layer_Version.objects.filter(layer=layer)
1170                _log("LAYER_ORPHAN_CHECK:Ref Count for '%s' = %d" % (layer.name,len(layer_refs)))
1171                if 0 == len(layer_refs):
1172                    _log("LAYER_ORPHAN_CHECK:DELETE orpahned '%s'" % (layer.name))
1173                    Layer.objects.filter(pk=layer.id).delete()
1174
1175        except Project.DoesNotExist:
1176            return error_response("Project %s does not exist" %
1177                                  kwargs['project_id'])
1178
1179        return JsonResponse({
1180            "error": "ok",
1181            "gotoUrl": reverse("all-projects", args=[])
1182        })
1183
1184
1185class XhrBuild(View):
1186    """ Delete a build object
1187
1188    Entry point: /xhr_build/<build_id>
1189    """
1190    def delete(self, request, *args, **kwargs):
1191        """
1192          Delete build data
1193
1194          Args:
1195              build_id = build_id
1196
1197          Returns:
1198              {"error": "ok"}
1199            or
1200              {"error": <error message>}
1201        """
1202        try:
1203            build = Build.objects.get(pk=kwargs['build_id'])
1204            project = build.project
1205            build.delete()
1206        except Build.DoesNotExist:
1207            return error_response("Build %s does not exist" %
1208                                  kwargs['build_id'])
1209        return JsonResponse({
1210            "error": "ok",
1211            "gotoUrl": reverse("projectbuilds", args=(project.pk,))
1212        })
1213