1# 2# BitBake Toaster Implementation 3# 4# Copyright (C) 2016 Intel Corporation 5# 6# SPDX-License-Identifier: GPL-2.0-only 7# 8# Please run flake8 on this file before sending patches 9 10import os 11import re 12import logging 13import json 14import subprocess 15from collections import Counter 16 17from orm.models import Project, ProjectTarget, Build, Layer_Version 18from orm.models import LayerVersionDependency, LayerSource, ProjectLayer 19from orm.models import Recipe, CustomImageRecipe, CustomImagePackage 20from orm.models import Layer, Target, Package, Package_Dependency 21from orm.models import ProjectVariable 22from bldcontrol.models import BuildRequest, BuildEnvironment 23from bldcontrol import bbcontroller 24 25from django.http import HttpResponse, JsonResponse 26from django.views.generic import View 27from django.urls import reverse 28from django.db.models import Q, F 29from django.db import Error 30from toastergui.templatetags.projecttags import filtered_filesizeformat 31 32# development/debugging support 33verbose = 2 34def _log(msg): 35 if 1 == verbose: 36 print(msg) 37 elif 2 == verbose: 38 f1=open('/tmp/toaster.log', 'a') 39 f1.write("|" + msg + "|\n" ) 40 f1.close() 41 42logger = logging.getLogger("toaster") 43 44 45def error_response(error): 46 return JsonResponse({"error": error}) 47 48 49class XhrBuildRequest(View): 50 51 def get(self, request, *args, **kwargs): 52 return HttpResponse() 53 54 @staticmethod 55 def cancel_build(br): 56 """Cancel a build request""" 57 try: 58 bbctrl = bbcontroller.BitbakeController(br.environment) 59 bbctrl.forceShutDown() 60 except: 61 # We catch a bunch of exceptions here because 62 # this is where the server has not had time to start up 63 # and the build request or build is in transit between 64 # processes. 65 # We can safely just set the build as cancelled 66 # already as it never got started 67 build = br.build 68 build.outcome = Build.CANCELLED 69 build.save() 70 71 # We now hand over to the buildinfohelper to update the 72 # build state once we've finished cancelling 73 br.state = BuildRequest.REQ_CANCELLING 74 br.save() 75 76 def post(self, request, *args, **kwargs): 77 """ 78 Build control 79 80 Entry point: /xhr_buildrequest/<project_id> 81 Method: POST 82 83 Args: 84 id: id of build to change 85 buildCancel = build_request_id ... 86 buildDelete = id ... 87 targets = recipe_name ... 88 89 Returns: 90 {"error": "ok"} 91 or 92 {"error": <error message>} 93 """ 94 95 project = Project.objects.get(pk=kwargs['pid']) 96 97 if 'buildCancel' in request.POST: 98 for i in request.POST['buildCancel'].strip().split(" "): 99 try: 100 br = BuildRequest.objects.get(project=project, pk=i) 101 self.cancel_build(br) 102 except BuildRequest.DoesNotExist: 103 return error_response('No such build request id %s' % i) 104 105 return error_response('ok') 106 107 if 'buildDelete' in request.POST: 108 for i in request.POST['buildDelete'].strip().split(" "): 109 try: 110 BuildRequest.objects.select_for_update().get( 111 project=project, 112 pk=i, 113 state__lte=BuildRequest.REQ_DELETED).delete() 114 115 except BuildRequest.DoesNotExist: 116 pass 117 return error_response("ok") 118 119 if 'targets' in request.POST: 120 ProjectTarget.objects.filter(project=project).delete() 121 s = str(request.POST['targets']) 122 for t in re.sub(r'[;%|"]', '', s).split(" "): 123 if ":" in t: 124 target, task = t.split(":") 125 else: 126 target = t 127 task = "" 128 ProjectTarget.objects.create(project=project, 129 target=target, 130 task=task) 131 project.schedule_build() 132 133 return error_response('ok') 134 135 response = HttpResponse() 136 response.status_code = 500 137 return response 138 139 140class XhrProjectUpdate(View): 141 142 def get(self, request, *args, **kwargs): 143 return HttpResponse() 144 145 def post(self, request, *args, **kwargs): 146 """ 147 Project Update 148 149 Entry point: /xhr_projectupdate/<project_id> 150 Method: POST 151 152 Args: 153 pid: pid of project to update 154 155 Returns: 156 {"error": "ok"} 157 or 158 {"error": <error message>} 159 """ 160 161 project = Project.objects.get(pk=kwargs['pid']) 162 logger.debug("ProjectUpdateCallback:project.pk=%d,project.builddir=%s" % (project.pk,project.builddir)) 163 164 if 'do_update' in request.POST: 165 166 # Extract any default image recipe 167 if 'default_image' in request.POST: 168 project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,str(request.POST['default_image'])) 169 else: 170 project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,'') 171 172 logger.debug("ProjectUpdateCallback:Chain to the build request") 173 174 # Chain to the build request 175 xhrBuildRequest = XhrBuildRequest() 176 return xhrBuildRequest.post(request, *args, **kwargs) 177 178 logger.warning("ERROR:XhrProjectUpdate") 179 response = HttpResponse() 180 response.status_code = 500 181 return response 182 183class XhrSetDefaultImageUrl(View): 184 185 def get(self, request, *args, **kwargs): 186 return HttpResponse() 187 188 def post(self, request, *args, **kwargs): 189 """ 190 Project Update 191 192 Entry point: /xhr_setdefaultimage/<project_id> 193 Method: POST 194 195 Args: 196 pid: pid of project to update default image 197 198 Returns: 199 {"error": "ok"} 200 or 201 {"error": <error message>} 202 """ 203 204 project = Project.objects.get(pk=kwargs['pid']) 205 logger.debug("XhrSetDefaultImageUrl:project.pk=%d" % (project.pk)) 206 207 # set any default image recipe 208 if 'targets' in request.POST: 209 default_target = str(request.POST['targets']) 210 project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,default_target) 211 logger.debug("XhrSetDefaultImageUrl,project.pk=%d,project.builddir=%s" % (project.pk,project.builddir)) 212 return error_response('ok') 213 214 logger.warning("ERROR:XhrSetDefaultImageUrl") 215 response = HttpResponse() 216 response.status_code = 500 217 return response 218 219 220# 221# Layer Management 222# 223# Rules for 'local_source_dir' layers 224# * Layers must have a unique name in the Layers table 225# * A 'local_source_dir' layer is supposed to be shared 226# by all projects that use it, so that it can have the 227# same logical name 228# * Each project that uses a layer will have its own 229# LayerVersion and Project Layer for it 230# * During the Paroject delete process, when the last 231# LayerVersion for a 'local_source_dir' layer is deleted 232# then the Layer record is deleted to remove orphans 233# 234 235def scan_layer_content(layer,layer_version): 236 # if this is a local layer directory, we can immediately scan its content 237 if layer.local_source_dir: 238 try: 239 # recipes-*/*/*.bb 240 cmd = '%s %s' % ('ls', os.path.join(layer.local_source_dir,'recipes-*/*/*.bb')) 241 recipes_list = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read() 242 recipes_list = recipes_list.decode("utf-8").strip() 243 if recipes_list and 'No such' not in recipes_list: 244 for recipe in recipes_list.split('\n'): 245 recipe_path = recipe[recipe.rfind('recipes-'):] 246 recipe_name = recipe[recipe.rfind('/')+1:].replace('.bb','') 247 recipe_ver = recipe_name.rfind('_') 248 if recipe_ver > 0: 249 recipe_name = recipe_name[0:recipe_ver] 250 if recipe_name: 251 ro, created = Recipe.objects.get_or_create( 252 layer_version=layer_version, 253 name=recipe_name 254 ) 255 if created: 256 ro.file_path = recipe_path 257 ro.summary = 'Recipe %s from layer %s' % (recipe_name,layer.name) 258 ro.description = ro.summary 259 ro.save() 260 261 except Exception as e: 262 logger.warning("ERROR:scan_layer_content: %s" % e) 263 264class XhrLayer(View): 265 """ Delete, Get, Add and Update Layer information 266 267 Methods: GET POST DELETE PUT 268 """ 269 270 def get(self, request, *args, **kwargs): 271 """ 272 Get layer information 273 274 Method: GET 275 Entry point: /xhr_layer/<project id>/<layerversion_id> 276 """ 277 278 try: 279 layer_version = Layer_Version.objects.get( 280 pk=kwargs['layerversion_id']) 281 282 project = Project.objects.get(pk=kwargs['pid']) 283 284 project_layers = ProjectLayer.objects.filter( 285 project=project).values_list("layercommit_id", 286 flat=True) 287 288 ret = { 289 'error': 'ok', 290 'id': layer_version.pk, 291 'name': layer_version.layer.name, 292 'layerdetailurl': 293 layer_version.get_detailspage_url(project.pk), 294 'vcs_ref': layer_version.get_vcs_reference(), 295 'vcs_url': layer_version.layer.vcs_url, 296 'local_source_dir': layer_version.layer.local_source_dir, 297 'layerdeps': { 298 "list": [ 299 { 300 "id": dep.id, 301 "name": dep.layer.name, 302 "layerdetailurl": 303 dep.get_detailspage_url(project.pk), 304 "vcs_url": dep.layer.vcs_url, 305 "vcs_reference": dep.get_vcs_reference() 306 } 307 for dep in layer_version.get_alldeps(project.id)] 308 }, 309 'projectlayers': list(project_layers) 310 } 311 312 return JsonResponse(ret) 313 except Layer_Version.DoesNotExist: 314 error_response("No such layer") 315 316 def post(self, request, *args, **kwargs): 317 """ 318 Update a layer 319 320 Method: POST 321 Entry point: /xhr_layer/<layerversion_id> 322 323 Args: 324 vcs_url, dirpath, commit, up_branch, summary, description, 325 local_source_dir 326 327 add_dep = append a layerversion_id as a dependency 328 rm_dep = remove a layerversion_id as a depedency 329 Returns: 330 {"error": "ok"} 331 or 332 {"error": <error message>} 333 """ 334 335 try: 336 # We currently only allow Imported layers to be edited 337 layer_version = Layer_Version.objects.get( 338 id=kwargs['layerversion_id'], 339 project=kwargs['pid'], 340 layer_source=LayerSource.TYPE_IMPORTED) 341 342 except Layer_Version.DoesNotExist: 343 return error_response("Cannot find imported layer to update") 344 345 if "vcs_url" in request.POST: 346 layer_version.layer.vcs_url = request.POST["vcs_url"] 347 if "dirpath" in request.POST: 348 layer_version.dirpath = request.POST["dirpath"] 349 if "commit" in request.POST: 350 layer_version.commit = request.POST["commit"] 351 layer_version.branch = request.POST["commit"] 352 if "summary" in request.POST: 353 layer_version.layer.summary = request.POST["summary"] 354 if "description" in request.POST: 355 layer_version.layer.description = request.POST["description"] 356 if "local_source_dir" in request.POST: 357 layer_version.layer.local_source_dir = \ 358 request.POST["local_source_dir"] 359 360 if "add_dep" in request.POST: 361 lvd = LayerVersionDependency( 362 layer_version=layer_version, 363 depends_on_id=request.POST["add_dep"]) 364 lvd.save() 365 366 if "rm_dep" in request.POST: 367 rm_dep = LayerVersionDependency.objects.get( 368 layer_version=layer_version, 369 depends_on_id=request.POST["rm_dep"]) 370 rm_dep.delete() 371 372 try: 373 layer_version.layer.save() 374 layer_version.save() 375 except Exception as e: 376 return error_response("Could not update layer version entry: %s" 377 % e) 378 379 return error_response("ok") 380 381 def put(self, request, *args, **kwargs): 382 """ Add a new layer 383 384 Method: PUT 385 Entry point: /xhr_layer/<project id>/ 386 Args: 387 project_id, name, 388 [vcs_url, dir_path, git_ref], [local_source_dir], [layer_deps 389 (csv)] 390 391 """ 392 393 try: 394 project = Project.objects.get(pk=kwargs['pid']) 395 396 layer_data = json.loads(request.body.decode('utf-8')) 397 398 # We require a unique layer name as otherwise the lists of layers 399 # becomes very confusing 400 existing_layers = \ 401 project.get_all_compatible_layer_versions().values_list( 402 "layer__name", 403 flat=True) 404 405 add_to_project = False 406 layer_deps_added = [] 407 if 'add_to_project' in layer_data: 408 add_to_project = True 409 410 if layer_data['name'] in existing_layers: 411 return JsonResponse({"error": "layer-name-exists"}) 412 413 if ('local_source_dir' in layer_data): 414 # Local layer can be shared across projects. They have no 'release' 415 # and are not included in get_all_compatible_layer_versions() above 416 layer,created = Layer.objects.get_or_create(name=layer_data['name']) 417 _log("Local Layer created=%s" % created) 418 else: 419 layer = Layer.objects.create(name=layer_data['name']) 420 421 layer_version = Layer_Version.objects.create( 422 layer=layer, 423 project=project, 424 layer_source=LayerSource.TYPE_IMPORTED) 425 426 # Local layer 427 if ('local_source_dir' in layer_data): ### and layer.local_source_dir: 428 layer.local_source_dir = layer_data['local_source_dir'] 429 # git layer 430 elif 'vcs_url' in layer_data: 431 layer.vcs_url = layer_data['vcs_url'] 432 layer_version.dirpath = layer_data['dir_path'] 433 layer_version.commit = layer_data['git_ref'] 434 layer_version.branch = layer_data['git_ref'] 435 436 layer.save() 437 layer_version.save() 438 439 if add_to_project: 440 ProjectLayer.objects.get_or_create( 441 layercommit=layer_version, project=project) 442 443 # Add the layer dependencies 444 if 'layer_deps' in layer_data: 445 for layer_dep_id in layer_data['layer_deps'].split(","): 446 layer_dep = Layer_Version.objects.get(pk=layer_dep_id) 447 LayerVersionDependency.objects.get_or_create( 448 layer_version=layer_version, depends_on=layer_dep) 449 450 # Add layer deps to the project if specified 451 if add_to_project: 452 created, pl = ProjectLayer.objects.get_or_create( 453 layercommit=layer_dep, project=project) 454 layer_deps_added.append( 455 {'name': layer_dep.layer.name, 456 'layerdetailurl': 457 layer_dep.get_detailspage_url(project.pk)}) 458 459 # Scan the layer's content and update components 460 scan_layer_content(layer,layer_version) 461 462 except Layer_Version.DoesNotExist: 463 return error_response("layer-dep-not-found") 464 except Project.DoesNotExist: 465 return error_response("project-not-found") 466 except KeyError: 467 return error_response("incorrect-parameters") 468 469 return JsonResponse({'error': "ok", 470 'imported_layer': { 471 'name': layer.name, 472 'layerdetailurl': 473 layer_version.get_detailspage_url()}, 474 'deps_added': layer_deps_added}) 475 476 def delete(self, request, *args, **kwargs): 477 """ Delete an imported layer 478 479 Method: DELETE 480 Entry point: /xhr_layer/<projed id>/<layerversion_id> 481 482 """ 483 try: 484 # We currently only allow Imported layers to be deleted 485 layer_version = Layer_Version.objects.get( 486 id=kwargs['layerversion_id'], 487 project=kwargs['pid'], 488 layer_source=LayerSource.TYPE_IMPORTED) 489 except Layer_Version.DoesNotExist: 490 return error_response("Cannot find imported layer to delete") 491 492 try: 493 ProjectLayer.objects.get(project=kwargs['pid'], 494 layercommit=layer_version).delete() 495 except ProjectLayer.DoesNotExist: 496 pass 497 498 layer_version.layer.delete() 499 layer_version.delete() 500 501 return JsonResponse({ 502 "error": "ok", 503 "gotoUrl": reverse('projectlayers', args=(kwargs['pid'],)) 504 }) 505 506 507class XhrCustomRecipe(View): 508 """ Create a custom image recipe """ 509 510 def post(self, request, *args, **kwargs): 511 """ 512 Custom image recipe REST API 513 514 Entry point: /xhr_customrecipe/ 515 Method: POST 516 517 Args: 518 name: name of custom recipe to create 519 project: target project id of orm.models.Project 520 base: base recipe id of orm.models.Recipe 521 522 Returns: 523 {"error": "ok", 524 "url": <url of the created recipe>} 525 or 526 {"error": <error message>} 527 """ 528 # check if request has all required parameters 529 for param in ('name', 'project', 'base'): 530 if param not in request.POST: 531 return error_response("Missing parameter '%s'" % param) 532 533 # get project and baserecipe objects 534 params = {} 535 for name, model in [("project", Project), 536 ("base", Recipe)]: 537 value = request.POST[name] 538 try: 539 params[name] = model.objects.get(id=value) 540 except model.DoesNotExist: 541 return error_response("Invalid %s id %s" % (name, value)) 542 543 # create custom recipe 544 try: 545 546 # Only allowed chars in name are a-z, 0-9 and - 547 if re.search(r'[^a-z|0-9|-]', request.POST["name"]): 548 return error_response("invalid-name") 549 550 custom_images = CustomImageRecipe.objects.all() 551 552 # Are there any recipes with this name already in our project? 553 existing_image_recipes_in_project = custom_images.filter( 554 name=request.POST["name"], project=params["project"]) 555 556 if existing_image_recipes_in_project.count() > 0: 557 return error_response("image-already-exists") 558 559 # Are there any recipes with this name which aren't custom 560 # image recipes? 561 custom_image_ids = custom_images.values_list('id', flat=True) 562 existing_non_image_recipes = Recipe.objects.filter( 563 Q(name=request.POST["name"]) & ~Q(pk__in=custom_image_ids) 564 ) 565 566 if existing_non_image_recipes.count() > 0: 567 return error_response("recipe-already-exists") 568 569 # create layer 'Custom layer' and verion if needed 570 layer, l_created = Layer.objects.get_or_create( 571 name=CustomImageRecipe.LAYER_NAME, 572 summary="Layer for custom recipes") 573 574 if l_created: 575 layer.local_source_dir = "toaster_created_layer" 576 layer.save() 577 578 # Check if we have a layer version already 579 # We don't use get_or_create here because the dirpath will change 580 # and is a required field 581 lver = Layer_Version.objects.filter(Q(project=params['project']) & 582 Q(layer=layer) & 583 Q(build=None)).last() 584 if lver is None: 585 lver, lv_created = Layer_Version.objects.get_or_create( 586 project=params['project'], 587 layer=layer, 588 layer_source=LayerSource.TYPE_LOCAL, 589 dirpath="toaster_created_layer") 590 591 # Add a dependency on our layer to the base recipe's layer 592 LayerVersionDependency.objects.get_or_create( 593 layer_version=lver, 594 depends_on=params["base"].layer_version) 595 596 # Add it to our current project if needed 597 ProjectLayer.objects.get_or_create(project=params['project'], 598 layercommit=lver, 599 optional=False) 600 601 # Create the actual recipe 602 recipe, r_created = CustomImageRecipe.objects.get_or_create( 603 name=request.POST["name"], 604 base_recipe=params["base"], 605 project=params["project"], 606 layer_version=lver, 607 is_image=True) 608 609 # If we created the object then setup these fields. They may get 610 # overwritten later on and cause the get_or_create to create a 611 # duplicate if they've changed. 612 if r_created: 613 recipe.file_path = request.POST["name"] 614 recipe.license = "MIT" 615 recipe.version = "0.1" 616 recipe.save() 617 618 except Error as err: 619 return error_response("Can't create custom recipe: %s" % err) 620 621 # Find the package list from the last build of this recipe/target 622 target = Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) & 623 Q(build__project=params['project']) & 624 (Q(target=params['base'].name) | 625 Q(target=recipe.name))).last() 626 if target: 627 # Copy in every package 628 # We don't want these packages to be linked to anything because 629 # that underlying data may change e.g. delete a build 630 for tpackage in target.target_installed_package_set.all(): 631 try: 632 built_package = tpackage.package 633 # The package had no recipe information so is a ghost 634 # package skip it 635 if built_package.recipe is None: 636 continue 637 638 config_package = CustomImagePackage.objects.get( 639 name=built_package.name) 640 641 recipe.includes_set.add(config_package) 642 except Exception as e: 643 logger.warning("Error adding package %s %s" % 644 (tpackage.package.name, e)) 645 pass 646 647 # pre-create layer directory structure, so that other builds 648 # are not blocked by this new recipe dependecy 649 # NOTE: this is parallel code to 'localhostbecontroller.py' 650 be = BuildEnvironment.objects.all()[0] 651 layerpath = os.path.join(be.builddir, 652 CustomImageRecipe.LAYER_NAME) 653 for name in ("conf", "recipes"): 654 path = os.path.join(layerpath, name) 655 if not os.path.isdir(path): 656 os.makedirs(path) 657 # pre-create layer.conf 658 config = os.path.join(layerpath, "conf", "layer.conf") 659 if not os.path.isfile(config): 660 with open(config, "w") as conf: 661 conf.write('BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n') 662 # pre-create new image's recipe file 663 recipe_path = os.path.join(layerpath, "recipes", "%s.bb" % 664 recipe.name) 665 with open(recipe_path, "w") as recipef: 666 content = recipe.generate_recipe_file_contents() 667 if not content: 668 # Delete this incomplete image recipe object 669 recipe.delete() 670 return error_response("recipe-parent-not-exist") 671 else: 672 recipef.write(recipe.generate_recipe_file_contents()) 673 674 return JsonResponse( 675 {"error": "ok", 676 "packages": recipe.get_all_packages().count(), 677 "url": reverse('customrecipe', args=(params['project'].pk, 678 recipe.id))}) 679 680 681class XhrCustomRecipeId(View): 682 """ 683 Set of ReST API processors working with recipe id. 684 685 Entry point: /xhr_customrecipe/<recipe_id> 686 687 Methods: 688 GET - Get details of custom image recipe 689 DELETE - Delete custom image recipe 690 691 Returns: 692 GET: 693 {"error": "ok", 694 "info": dictionary of field name -> value pairs 695 of the CustomImageRecipe model} 696 DELETE: 697 {"error": "ok"} 698 or 699 {"error": <error message>} 700 """ 701 @staticmethod 702 def _get_ci_recipe(recipe_id): 703 """ Get Custom Image recipe or return an error response""" 704 try: 705 custom_recipe = \ 706 CustomImageRecipe.objects.get(pk=recipe_id) 707 return custom_recipe, None 708 709 except CustomImageRecipe.DoesNotExist: 710 return None, error_response("Custom recipe with id=%s " 711 "not found" % recipe_id) 712 713 def get(self, request, *args, **kwargs): 714 custom_recipe, error = self._get_ci_recipe(kwargs['recipe_id']) 715 if error: 716 return error 717 718 if request.method == 'GET': 719 info = {"id": custom_recipe.id, 720 "name": custom_recipe.name, 721 "base_recipe_id": custom_recipe.base_recipe.id, 722 "project_id": custom_recipe.project.id} 723 724 return JsonResponse({"error": "ok", "info": info}) 725 726 def delete(self, request, *args, **kwargs): 727 custom_recipe, error = self._get_ci_recipe(kwargs['recipe_id']) 728 if error: 729 return error 730 731 project = custom_recipe.project 732 733 custom_recipe.delete() 734 return JsonResponse({"error": "ok", 735 "gotoUrl": reverse("projectcustomimages", 736 args=(project.pk,))}) 737 738 739class XhrCustomRecipePackages(View): 740 """ 741 ReST API to add/remove packages to/from custom recipe. 742 743 Entry point: /xhr_customrecipe/<recipe_id>/packages/<package_id> 744 Methods: 745 PUT - Add package to the recipe 746 DELETE - Delete package from the recipe 747 GET - Get package information 748 749 Returns: 750 {"error": "ok"} 751 or 752 {"error": <error message>} 753 """ 754 @staticmethod 755 def _get_package(package_id): 756 try: 757 package = CustomImagePackage.objects.get(pk=package_id) 758 return package, None 759 except Package.DoesNotExist: 760 return None, error_response("Package with id=%s " 761 "not found" % package_id) 762 763 def _traverse_dependents(self, next_package_id, 764 rev_deps, all_current_packages, tree_level=0): 765 """ 766 Recurse through reverse dependency tree for next_package_id. 767 Limit the reverse dependency search to packages not already scanned, 768 that is, not already in rev_deps. 769 Limit the scan to a depth (tree_level) not exceeding the count of 770 all packages in the custom image, and if that depth is exceeded 771 return False, pop out of the recursion, and write a warning 772 to the log, but this is unlikely, suggesting a dependency loop 773 not caught by bitbake. 774 On return, the input/output arg rev_deps is appended with queryset 775 dictionary elements, annotated for use in the customimage template. 776 The list has unsorted, but unique elements. 777 """ 778 max_dependency_tree_depth = all_current_packages.count() 779 if tree_level >= max_dependency_tree_depth: 780 logger.warning( 781 "The number of reverse dependencies " 782 "for this package exceeds " + max_dependency_tree_depth + 783 " and the remaining reverse dependencies will not be removed") 784 return True 785 786 package = CustomImagePackage.objects.get(id=next_package_id) 787 dependents = \ 788 package.package_dependencies_target.annotate( 789 name=F('package__name'), 790 pk=F('package__pk'), 791 size=F('package__size'), 792 ).values("name", "pk", "size").exclude( 793 ~Q(pk__in=all_current_packages) 794 ) 795 796 for pkg in dependents: 797 if pkg in rev_deps: 798 # already seen, skip dependent search 799 continue 800 801 rev_deps.append(pkg) 802 if (self._traverse_dependents(pkg["pk"], rev_deps, 803 all_current_packages, 804 tree_level+1)): 805 return True 806 807 return False 808 809 def _get_all_dependents(self, package_id, all_current_packages): 810 """ 811 Returns sorted list of recursive reverse dependencies for package_id, 812 as a list of dictionary items, by recursing through dependency 813 relationships. 814 """ 815 rev_deps = [] 816 self._traverse_dependents(package_id, rev_deps, all_current_packages) 817 rev_deps = sorted(rev_deps, key=lambda x: x["name"]) 818 return rev_deps 819 820 def get(self, request, *args, **kwargs): 821 recipe, error = XhrCustomRecipeId._get_ci_recipe( 822 kwargs['recipe_id']) 823 if error: 824 return error 825 826 # If no package_id then list all the current packages 827 if not kwargs['package_id']: 828 total_size = 0 829 packages = recipe.get_all_packages().values("id", 830 "name", 831 "version", 832 "size") 833 for package in packages: 834 package['size_formatted'] = \ 835 filtered_filesizeformat(package['size']) 836 total_size += package['size'] 837 838 return JsonResponse({"error": "ok", 839 "packages": list(packages), 840 "total": len(packages), 841 "total_size": total_size, 842 "total_size_formatted": 843 filtered_filesizeformat(total_size)}) 844 else: 845 package, error = XhrCustomRecipePackages._get_package( 846 kwargs['package_id']) 847 if error: 848 return error 849 850 all_current_packages = recipe.get_all_packages() 851 852 # Dependencies for package which aren't satisfied by the 853 # current packages in the custom image recipe 854 deps = package.package_dependencies_source.for_target_or_none( 855 recipe.name)['packages'].annotate( 856 name=F('depends_on__name'), 857 pk=F('depends_on__pk'), 858 size=F('depends_on__size'), 859 ).values("name", "pk", "size").filter( 860 # There are two depends types we don't know why 861 (Q(dep_type=Package_Dependency.TYPE_TRDEPENDS) | 862 Q(dep_type=Package_Dependency.TYPE_RDEPENDS)) & 863 ~Q(pk__in=all_current_packages) 864 ) 865 866 # Reverse dependencies which are needed by packages that are 867 # in the image. Recursive search providing all dependents, 868 # not just immediate dependents. 869 reverse_deps = self._get_all_dependents(kwargs['package_id'], 870 all_current_packages) 871 total_size_deps = 0 872 total_size_reverse_deps = 0 873 874 for dep in deps: 875 dep['size_formatted'] = \ 876 filtered_filesizeformat(dep['size']) 877 total_size_deps += dep['size'] 878 879 for dep in reverse_deps: 880 dep['size_formatted'] = \ 881 filtered_filesizeformat(dep['size']) 882 total_size_reverse_deps += dep['size'] 883 884 return JsonResponse( 885 {"error": "ok", 886 "id": package.pk, 887 "name": package.name, 888 "version": package.version, 889 "unsatisfied_dependencies": list(deps), 890 "unsatisfied_dependencies_size": total_size_deps, 891 "unsatisfied_dependencies_size_formatted": 892 filtered_filesizeformat(total_size_deps), 893 "reverse_dependencies": list(reverse_deps), 894 "reverse_dependencies_size": total_size_reverse_deps, 895 "reverse_dependencies_size_formatted": 896 filtered_filesizeformat(total_size_reverse_deps)}) 897 898 def put(self, request, *args, **kwargs): 899 recipe, error = XhrCustomRecipeId._get_ci_recipe(kwargs['recipe_id']) 900 package, error = self._get_package(kwargs['package_id']) 901 if error: 902 return error 903 904 included_packages = recipe.includes_set.values_list('pk', 905 flat=True) 906 907 # If we're adding back a package which used to be included in this 908 # image all we need to do is remove it from the excludes 909 if package.pk in included_packages: 910 try: 911 recipe.excludes_set.remove(package) 912 return {"error": "ok"} 913 except Package.DoesNotExist: 914 return error_response("Package %s not found in excludes" 915 " but was in included list" % 916 package.name) 917 else: 918 recipe.appends_set.add(package) 919 # Make sure that package is not in the excludes set 920 try: 921 recipe.excludes_set.remove(package) 922 except: 923 pass 924 925 # Add the dependencies we think will be added to the recipe 926 # as a result of appending this package. 927 # TODO this should recurse down the entire deps tree 928 for dep in package.package_dependencies_source.all_depends(): 929 try: 930 cust_package = CustomImagePackage.objects.get( 931 name=dep.depends_on.name) 932 933 recipe.includes_set.add(cust_package) 934 try: 935 # When adding the pre-requisite package, make 936 # sure it's not in the excluded list from a 937 # prior removal. 938 recipe.excludes_set.remove(cust_package) 939 except package.DoesNotExist: 940 # Don't care if the package had never been excluded 941 pass 942 except: 943 logger.warning("Could not add package's suggested" 944 "dependencies to the list") 945 return JsonResponse({"error": "ok"}) 946 947 def delete(self, request, *args, **kwargs): 948 recipe, error = XhrCustomRecipeId._get_ci_recipe(kwargs['recipe_id']) 949 package, error = self._get_package(kwargs['package_id']) 950 if error: 951 return error 952 953 try: 954 included_packages = recipe.includes_set.values_list('pk', 955 flat=True) 956 # If we're deleting a package which is included we need to 957 # Add it to the excludes list. 958 if package.pk in included_packages: 959 recipe.excludes_set.add(package) 960 else: 961 recipe.appends_set.remove(package) 962 963 # remove dependencies as well 964 all_current_packages = recipe.get_all_packages() 965 966 reverse_deps_dictlist = self._get_all_dependents( 967 package.pk, 968 all_current_packages) 969 970 ids = [entry['pk'] for entry in reverse_deps_dictlist] 971 reverse_deps = CustomImagePackage.objects.filter(id__in=ids) 972 for r in reverse_deps: 973 try: 974 if r.id in included_packages: 975 recipe.excludes_set.add(r) 976 else: 977 recipe.appends_set.remove(r) 978 except: 979 pass 980 981 return JsonResponse({"error": "ok"}) 982 except CustomImageRecipe.DoesNotExist: 983 return error_response("Tried to remove package that wasn't" 984 " present") 985 986 987class XhrProject(View): 988 """ Create, delete or edit a project 989 990 Entry point: /xhr_project/<project_id> 991 """ 992 def post(self, request, *args, **kwargs): 993 """ 994 Edit project control 995 996 Args: 997 layerAdd = layer_version_id layer_version_id ... 998 layerDel = layer_version_id layer_version_id ... 999 projectName = new_project_name 1000 machineName = new_machine_name 1001 1002 Returns: 1003 {"error": "ok"} 1004 or 1005 {"error": <error message>} 1006 """ 1007 try: 1008 prj = Project.objects.get(pk=kwargs['project_id']) 1009 except Project.DoesNotExist: 1010 return error_response("No such project") 1011 1012 # Add layers 1013 if 'layerAdd' in request.POST and len(request.POST['layerAdd']) > 0: 1014 for layer_version_id in request.POST['layerAdd'].split(','): 1015 try: 1016 lv = Layer_Version.objects.get(pk=int(layer_version_id)) 1017 ProjectLayer.objects.get_or_create(project=prj, 1018 layercommit=lv) 1019 except Layer_Version.DoesNotExist: 1020 return error_response("Layer version %s asked to add " 1021 "doesn't exist" % layer_version_id) 1022 1023 # Remove layers 1024 if 'layerDel' in request.POST and len(request.POST['layerDel']) > 0: 1025 layer_version_ids = request.POST['layerDel'].split(',') 1026 ProjectLayer.objects.filter( 1027 project=prj, 1028 layercommit_id__in=layer_version_ids).delete() 1029 1030 # Project name change 1031 if 'projectName' in request.POST: 1032 prj.name = request.POST['projectName'] 1033 prj.save() 1034 1035 # Machine name change 1036 if 'machineName' in request.POST: 1037 machinevar = prj.projectvariable_set.get(name="MACHINE") 1038 machinevar.value = request.POST['machineName'] 1039 machinevar.save() 1040 1041 # Distro name change 1042 if 'distroName' in request.POST: 1043 distrovar = prj.projectvariable_set.get(name="DISTRO") 1044 distrovar.value = request.POST['distroName'] 1045 distrovar.save() 1046 1047 return JsonResponse({"error": "ok"}) 1048 1049 def get(self, request, *args, **kwargs): 1050 """ 1051 Returns: 1052 json object representing the current project 1053 or: 1054 {"error": <error message>} 1055 """ 1056 1057 try: 1058 project = Project.objects.get(pk=kwargs['project_id']) 1059 except Project.DoesNotExist: 1060 return error_response("Project %s does not exist" % 1061 kwargs['project_id']) 1062 1063 # Create the frequently built targets list 1064 1065 freqtargets = Counter(Target.objects.filter( 1066 Q(build__project=project), 1067 ~Q(build__outcome=Build.IN_PROGRESS) 1068 ).order_by("target").values_list("target", flat=True)) 1069 1070 freqtargets = freqtargets.most_common(5) 1071 1072 # We now have the targets in order of frequency but if there are two 1073 # with the same frequency then we need to make sure those are in 1074 # alphabetical order without losing the frequency ordering 1075 1076 tmp = [] 1077 switch = None 1078 for i, freqtartget in enumerate(freqtargets): 1079 target, count = freqtartget 1080 try: 1081 target_next, count_next = freqtargets[i+1] 1082 if count == count_next and target > target_next: 1083 switch = target 1084 continue 1085 except IndexError: 1086 pass 1087 1088 tmp.append(target) 1089 1090 if switch: 1091 tmp.append(switch) 1092 switch = None 1093 1094 freqtargets = tmp 1095 1096 layers = [] 1097 for layer in project.projectlayer_set.all(): 1098 layers.append({ 1099 "id": layer.layercommit.pk, 1100 "name": layer.layercommit.layer.name, 1101 "vcs_url": layer.layercommit.layer.vcs_url, 1102 "local_source_dir": layer.layercommit.layer.local_source_dir, 1103 "vcs_reference": layer.layercommit.get_vcs_reference(), 1104 "url": layer.layercommit.layer.layer_index_url, 1105 "layerdetailurl": layer.layercommit.get_detailspage_url( 1106 project.pk), 1107 "xhrLayerUrl": reverse("xhr_layer", 1108 args=(project.pk, 1109 layer.layercommit.pk)), 1110 "layersource": layer.layercommit.layer_source 1111 }) 1112 1113 data = { 1114 "name": project.name, 1115 "layers": layers, 1116 "freqtargets": freqtargets, 1117 } 1118 1119 if project.release is not None: 1120 data['release'] = { 1121 "id": project.release.pk, 1122 "name": project.release.name, 1123 "description": project.release.description 1124 } 1125 1126 try: 1127 data["machine"] = {"name": 1128 project.projectvariable_set.get( 1129 name="MACHINE").value} 1130 except ProjectVariable.DoesNotExist: 1131 data["machine"] = None 1132 try: 1133 data["distro"] = {"name": 1134 project.projectvariable_set.get( 1135 name="DISTRO").value} 1136 except ProjectVariable.DoesNotExist: 1137 data["distro"] = None 1138 1139 data['error'] = "ok" 1140 1141 return JsonResponse(data) 1142 1143 def put(self, request, *args, **kwargs): 1144 # TODO create new project api 1145 return HttpResponse() 1146 1147 def delete(self, request, *args, **kwargs): 1148 """Delete a project. Cancels any builds in progress""" 1149 try: 1150 project = Project.objects.get(pk=kwargs['project_id']) 1151 # Cancel any builds in progress 1152 for br in BuildRequest.objects.filter( 1153 project=project, 1154 state=BuildRequest.REQ_INPROGRESS): 1155 XhrBuildRequest.cancel_build(br) 1156 1157 # gather potential orphaned local layers attached to this project 1158 project_local_layer_list = [] 1159 for pl in ProjectLayer.objects.filter(project=project): 1160 if pl.layercommit.layer_source == LayerSource.TYPE_IMPORTED: 1161 project_local_layer_list.append(pl.layercommit.layer) 1162 1163 # deep delete the project and its dependencies 1164 project.delete() 1165 1166 # delete any local layers now orphaned 1167 _log("LAYER_ORPHAN_CHECK:Check for orphaned layers") 1168 for layer in project_local_layer_list: 1169 layer_refs = Layer_Version.objects.filter(layer=layer) 1170 _log("LAYER_ORPHAN_CHECK:Ref Count for '%s' = %d" % (layer.name,len(layer_refs))) 1171 if 0 == len(layer_refs): 1172 _log("LAYER_ORPHAN_CHECK:DELETE orpahned '%s'" % (layer.name)) 1173 Layer.objects.filter(pk=layer.id).delete() 1174 1175 except Project.DoesNotExist: 1176 return error_response("Project %s does not exist" % 1177 kwargs['project_id']) 1178 1179 return JsonResponse({ 1180 "error": "ok", 1181 "gotoUrl": reverse("all-projects", args=[]) 1182 }) 1183 1184 1185class XhrBuild(View): 1186 """ Delete a build object 1187 1188 Entry point: /xhr_build/<build_id> 1189 """ 1190 def delete(self, request, *args, **kwargs): 1191 """ 1192 Delete build data 1193 1194 Args: 1195 build_id = build_id 1196 1197 Returns: 1198 {"error": "ok"} 1199 or 1200 {"error": <error message>} 1201 """ 1202 try: 1203 build = Build.objects.get(pk=kwargs['build_id']) 1204 project = build.project 1205 build.delete() 1206 except Build.DoesNotExist: 1207 return error_response("Build %s does not exist" % 1208 kwargs['build_id']) 1209 return JsonResponse({ 1210 "error": "ok", 1211 "gotoUrl": reverse("projectbuilds", args=(project.pk,)) 1212 }) 1213