1# 2# BitBake Toaster Implementation 3# 4# Copyright (C) 2013 Intel Corporation 5# 6# SPDX-License-Identifier: GPL-2.0-only 7# 8 9import ast 10import re 11import subprocess 12import sys 13 14import bb.cooker 15from bb.ui import toasterui 16from bb.ui import eventreplay 17 18from django.db.models import F, Q, Sum 19from django.db import IntegrityError 20from django.shortcuts import render, redirect, get_object_or_404, HttpResponseRedirect 21from django.utils.http import urlencode 22from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe 23from orm.models import LogMessage, Variable, Package_Dependency, Package 24from orm.models import Task_Dependency, Package_File 25from orm.models import Target_Installed_Package, Target_File 26from orm.models import TargetKernelFile, TargetSDKFile, Target_Image_File 27from orm.models import BitbakeVersion, CustomImageRecipe, EventLogsImports 28 29from django.urls import reverse, resolve 30from django.contrib import messages 31 32from django.core.exceptions import ObjectDoesNotExist 33from django.core.files.storage import FileSystemStorage 34from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile 35from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger 36from django.http import HttpResponseNotFound, JsonResponse 37from django.utils import timezone 38from django.views.generic import TemplateView 39from datetime import timedelta, datetime 40from toastergui.templatetags.projecttags import json as jsonfilter 41from decimal import Decimal 42import json 43import os 44from os.path import dirname 45import mimetypes 46 47from toastergui.forms import LoadFileForm 48 49from collections import namedtuple 50 51import logging 52 53from toastermain.logs import log_view_mixin 54 55logger = logging.getLogger("toaster") 56 57# Project creation and managed build enable 58project_enable = ('1' == os.environ.get('TOASTER_BUILDSERVER')) 59is_project_specific = ('1' == os.environ.get('TOASTER_PROJECTSPECIFIC')) 60import_page = False 61 62class MimeTypeFinder(object): 63 # setting this to False enables additional non-standard mimetypes 64 # to be included in the guess 65 _strict = False 66 67 # returns the mimetype for a file path as a string, 68 # or 'application/octet-stream' if the type couldn't be guessed 69 @classmethod 70 def get_mimetype(self, path): 71 guess = mimetypes.guess_type(path, self._strict) 72 guessed_type = guess[0] 73 if guessed_type is None: 74 guessed_type = 'application/octet-stream' 75 return guessed_type 76 77# single point to add global values into the context before rendering 78@log_view_mixin 79def toaster_render(request, page, context): 80 context['project_enable'] = project_enable 81 context['project_specific'] = is_project_specific 82 return render(request, page, context) 83 84 85# all new sessions should come through the landing page; 86# determine in which mode we are running in, and redirect appropriately 87def landing(request): 88 # in build mode, we redirect to the command-line builds page 89 # if there are any builds for the default (cli builds) project 90 default_project = Project.objects.get_or_create_default_project() 91 default_project_builds = Build.objects.filter(project = default_project) 92 93 # we only redirect to projects page if there is a user-generated project 94 num_builds = Build.objects.all().count() 95 user_projects = Project.objects.filter(is_default = False) 96 has_user_project = user_projects.count() > 0 97 98 if num_builds == 0 and has_user_project: 99 return redirect(reverse('all-projects'), permanent = False) 100 101 if num_builds > 0: 102 return redirect(reverse('all-builds'), permanent = False) 103 104 context = {'lvs_nos' : Layer_Version.objects.all().count()} 105 106 return toaster_render(request, 'landing.html', context) 107 108def objtojson(obj): 109 from django.db.models.query import QuerySet 110 from django.db.models import Model 111 112 if isinstance(obj, datetime): 113 return obj.isoformat() 114 elif isinstance(obj, timedelta): 115 return obj.total_seconds() 116 elif isinstance(obj, QuerySet) or isinstance(obj, set): 117 return list(obj) 118 elif isinstance(obj, Decimal): 119 return str(obj) 120 elif type(obj).__name__ == "RelatedManager": 121 return [x.pk for x in obj.all()] 122 elif hasattr( obj, '__dict__') and isinstance(obj, Model): 123 d = obj.__dict__ 124 nd = dict(d) 125 for di in d.keys(): 126 if di.startswith("_"): 127 del nd[di] 128 elif isinstance(d[di], Model): 129 nd[di] = d[di].pk 130 elif isinstance(d[di], int) and hasattr(obj, "get_%s_display" % di): 131 nd[di] = getattr(obj, "get_%s_display" % di)() 132 return nd 133 elif isinstance( obj, type(lambda x:x)): 134 import inspect 135 return inspect.getsourcelines(obj)[0] 136 else: 137 raise TypeError("Unserializable object %s (%s) of type %s" % ( obj, dir(obj), type(obj))) 138 139 140def _lv_to_dict(prj, x = None): 141 if x is None: 142 def wrapper(x): 143 return _lv_to_dict(prj, x) 144 return wrapper 145 146 return {"id": x.pk, 147 "name": x.layer.name, 148 "tooltip": "%s | %s" % (x.layer.vcs_url,x.get_vcs_reference()), 149 "detail": "(%s" % x.layer.vcs_url + (")" if x.release is None else " | "+x.get_vcs_reference()+")"), 150 "giturl": x.layer.vcs_url, 151 "layerdetailurl" : reverse('layerdetails', args=(prj.id,x.pk)), 152 "revision" : x.get_vcs_reference(), 153 } 154 155 156def _build_page_range(paginator, index = 1): 157 try: 158 page = paginator.page(index) 159 except PageNotAnInteger: 160 page = paginator.page(1) 161 except EmptyPage: 162 page = paginator.page(paginator.num_pages) 163 164 165 page.page_range = [page.number] 166 crt_range = 0 167 for i in range(1,5): 168 if (page.number + i) <= paginator.num_pages: 169 page.page_range = page.page_range + [ page.number + i] 170 crt_range +=1 171 if (page.number - i) > 0: 172 page.page_range = [page.number -i] + page.page_range 173 crt_range +=1 174 if crt_range == 4: 175 break 176 return page 177 178 179def _verify_parameters(g, mandatory_parameters): 180 miss = [] 181 for mp in mandatory_parameters: 182 if not mp in g: 183 miss.append(mp) 184 if len(miss): 185 return miss 186 return None 187 188def _redirect_parameters(view, g, mandatory_parameters, *args, **kwargs): 189 try: 190 from urllib import unquote, urlencode 191 except ImportError: 192 from urllib.parse import unquote, urlencode 193 url = reverse(view, kwargs=kwargs) 194 params = {} 195 for i in g: 196 params[i] = g[i] 197 for i in mandatory_parameters: 198 if not i in params: 199 params[i] = unquote(str(mandatory_parameters[i])) 200 201 return redirect(url + "?%s" % urlencode(params), permanent = False, **kwargs) 202 203class RedirectException(Exception): 204 def __init__(self, view, g, mandatory_parameters, *args, **kwargs): 205 super(RedirectException, self).__init__() 206 self.view = view 207 self.g = g 208 self.mandatory_parameters = mandatory_parameters 209 self.oargs = args 210 self.okwargs = kwargs 211 212 def get_redirect_response(self): 213 return _redirect_parameters(self.view, self.g, self.mandatory_parameters, self.oargs, **self.okwargs) 214 215FIELD_SEPARATOR = ":" 216AND_VALUE_SEPARATOR = "!" 217OR_VALUE_SEPARATOR = "|" 218DESCENDING = "-" 219 220def __get_q_for_val(name, value): 221 if "OR" in value or "AND" in value: 222 result = None 223 for x in value.split("OR"): 224 x = __get_q_for_val(name, x) 225 result = result | x if result else x 226 return result 227 if "AND" in value: 228 result = None 229 for x in value.split("AND"): 230 x = __get_q_for_val(name, x) 231 result = result & x if result else x 232 return result 233 if value.startswith("NOT"): 234 value = value[3:] 235 if value == 'None': 236 value = None 237 kwargs = { name : value } 238 return ~Q(**kwargs) 239 else: 240 if value == 'None': 241 value = None 242 kwargs = { name : value } 243 return Q(**kwargs) 244 245def _get_filtering_query(filter_string): 246 247 search_terms = filter_string.split(FIELD_SEPARATOR) 248 and_keys = search_terms[0].split(AND_VALUE_SEPARATOR) 249 and_values = search_terms[1].split(AND_VALUE_SEPARATOR) 250 251 and_query = None 252 for kv in zip(and_keys, and_values): 253 or_keys = kv[0].split(OR_VALUE_SEPARATOR) 254 or_values = kv[1].split(OR_VALUE_SEPARATOR) 255 query = None 256 for key, val in zip(or_keys, or_values): 257 x = __get_q_for_val(key, val) 258 query = query | x if query else x 259 260 and_query = and_query & query if and_query else query 261 262 return and_query 263 264def _get_toggle_order(request, orderkey, toggle_reverse = False): 265 if toggle_reverse: 266 return "%s:+" % orderkey if request.GET.get('orderby', "") == "%s:-" % orderkey else "%s:-" % orderkey 267 else: 268 return "%s:-" % orderkey if request.GET.get('orderby', "") == "%s:+" % orderkey else "%s:+" % orderkey 269 270def _get_toggle_order_icon(request, orderkey): 271 if request.GET.get('orderby', "") == "%s:+"%orderkey: 272 return "down" 273 elif request.GET.get('orderby', "") == "%s:-"%orderkey: 274 return "up" 275 else: 276 return None 277 278# we check that the input comes in a valid form that we can recognize 279def _validate_input(field_input, model): 280 281 invalid = None 282 283 if field_input: 284 field_input_list = field_input.split(FIELD_SEPARATOR) 285 286 # Check we have only one colon 287 if len(field_input_list) != 2: 288 invalid = "We have an invalid number of separators: " + field_input + " -> " + str(field_input_list) 289 return None, invalid 290 291 # Check we have an equal number of terms both sides of the colon 292 if len(field_input_list[0].split(AND_VALUE_SEPARATOR)) != len(field_input_list[1].split(AND_VALUE_SEPARATOR)): 293 invalid = "Not all arg names got values" 294 return None, invalid + str(field_input_list) 295 296 # Check we are looking for a valid field 297 valid_fields = [f.name for f in model._meta.get_fields()] 298 for field in field_input_list[0].split(AND_VALUE_SEPARATOR): 299 if True in [field.startswith(x) for x in valid_fields]: 300 break 301 else: 302 return None, (field, valid_fields) 303 304 return field_input, invalid 305 306# uses search_allowed_fields in orm/models.py to create a search query 307# for these fields with the supplied input text 308def _get_search_results(search_term, queryset, model): 309 search_object = None 310 for st in search_term.split(" "): 311 queries = None 312 for field in model.search_allowed_fields: 313 query = Q(**{field + '__icontains': st}) 314 queries = queries | query if queries else query 315 316 search_object = search_object & queries if search_object else queries 317 queryset = queryset.filter(search_object) 318 319 return queryset 320 321 322# function to extract the search/filter/ordering parameters from the request 323# it uses the request and the model to validate input for the filter and orderby values 324def _search_tuple(request, model): 325 ordering_string, invalid = _validate_input(request.GET.get('orderby', ''), model) 326 if invalid: 327 raise BaseException("Invalid ordering model:" + str(model) + str(invalid)) 328 329 filter_string, invalid = _validate_input(request.GET.get('filter', ''), model) 330 if invalid: 331 raise BaseException("Invalid filter " + str(invalid)) 332 333 search_term = request.GET.get('search', '') 334 return (filter_string, search_term, ordering_string) 335 336 337# returns a lazy-evaluated queryset for a filter/search/order combination 338def _get_queryset(model, queryset, filter_string, search_term, ordering_string, ordering_secondary=''): 339 if filter_string: 340 filter_query = _get_filtering_query(filter_string) 341 queryset = queryset.filter(filter_query) 342 else: 343 queryset = queryset.all() 344 345 if search_term: 346 queryset = _get_search_results(search_term, queryset, model) 347 348 if ordering_string: 349 column, order = ordering_string.split(':') 350 if column == re.sub('-','',ordering_secondary): 351 ordering_secondary='' 352 if order.lower() == DESCENDING: 353 column = '-' + column 354 if ordering_secondary: 355 queryset = queryset.order_by(column, ordering_secondary) 356 else: 357 queryset = queryset.order_by(column) 358 359 # insure only distinct records (e.g. from multiple search hits) are returned 360 return queryset.distinct() 361 362# returns the value of entries per page and the name of the applied sorting field. 363# if the value is given explicitly as a GET parameter it will be the first selected, 364# otherwise the cookie value will be used. 365def _get_parameters_values(request, default_count, default_order): 366 current_url = resolve(request.path_info).url_name 367 pagesize = request.GET.get('count', request.session.get('%s_count' % current_url, default_count)) 368 orderby = request.GET.get('orderby', request.session.get('%s_orderby' % current_url, default_order)) 369 return (pagesize, orderby) 370 371 372# set cookies for parameters. this is usefull in case parameters are set 373# manually from the GET values of the link 374def _set_parameters_values(pagesize, orderby, request): 375 from django.urls import resolve 376 current_url = resolve(request.path_info).url_name 377 request.session['%s_count' % current_url] = pagesize 378 request.session['%s_orderby' % current_url] =orderby 379 380# date range: normalize GUI's dd/mm/yyyy to date object 381def _normalize_input_date(date_str,default): 382 date_str=re.sub('/', '-', date_str) 383 # accept dd/mm/yyyy to d/m/yy 384 try: 385 date_in = datetime.strptime(date_str, "%d-%m-%Y") 386 except ValueError: 387 # courtesy try with two digit year 388 try: 389 date_in = datetime.strptime(date_str, "%d-%m-%y") 390 except ValueError: 391 return default 392 date_in = date_in.replace(tzinfo=default.tzinfo) 393 return date_in 394 395# convert and normalize any received date range filter, for example: 396# "completed_on__gte!completed_on__lt:01/03/2015!02/03/2015_daterange" to 397# "completed_on__gte!completed_on__lt:2015-03-01!2015-03-02" 398def _modify_date_range_filter(filter_string): 399 # was the date range radio button selected? 400 if 0 > filter_string.find('_daterange'): 401 return filter_string,'' 402 # normalize GUI dates to database format 403 filter_string = filter_string.replace('_daterange','').replace(':','!'); 404 filter_list = filter_string.split('!'); 405 if 4 != len(filter_list): 406 return filter_string 407 today = timezone.localtime(timezone.now()) 408 date_id = filter_list[1] 409 date_from = _normalize_input_date(filter_list[2],today) 410 date_to = _normalize_input_date(filter_list[3],today) 411 # swap dates if manually set dates are out of order 412 if date_to < date_from: 413 date_to,date_from = date_from,date_to 414 # convert to strings, make 'date_to' inclusive by moving to begining of next day 415 date_from_str = date_from.strftime("%Y-%m-%d") 416 date_to_str = (date_to+timedelta(days=1)).strftime("%Y-%m-%d") 417 filter_string=filter_list[0]+'!'+filter_list[1]+':'+date_from_str+'!'+date_to_str 418 daterange_selected = re.sub('__.*','', date_id) 419 return filter_string,daterange_selected 420 421def _add_daterange_context(queryset_all, request, daterange_list): 422 # calculate the exact begining of local today and yesterday 423 today_begin = timezone.localtime(timezone.now()) 424 yesterday_begin = today_begin - timedelta(days=1) 425 # add daterange persistent 426 context_date = {} 427 context_date['last_date_from'] = request.GET.get('last_date_from',timezone.localtime(timezone.now()).strftime("%d/%m/%Y")) 428 context_date['last_date_to' ] = request.GET.get('last_date_to' ,context_date['last_date_from']) 429 # calculate the date ranges, avoid second sort for 'created' 430 # fetch the respective max range from the database 431 context_date['daterange_filter']='' 432 for key in daterange_list: 433 queryset_key = queryset_all.order_by(key) 434 try: 435 context_date['dateMin_'+key]=timezone.localtime(getattr(queryset_key.first(),key)).strftime("%d/%m/%Y") 436 except AttributeError: 437 context_date['dateMin_'+key]=timezone.localtime(timezone.now()) 438 try: 439 context_date['dateMax_'+key]=timezone.localtime(getattr(queryset_key.last(),key)).strftime("%d/%m/%Y") 440 except AttributeError: 441 context_date['dateMax_'+key]=timezone.localtime(timezone.now()) 442 return context_date,today_begin,yesterday_begin 443 444 445## 446# build dashboard for a single build, coming in as argument 447# Each build may contain multiple targets and each target 448# may generate multiple image files. display them all. 449# 450def builddashboard( request, build_id ): 451 template = "builddashboard.html" 452 if Build.objects.filter( pk=build_id ).count( ) == 0 : 453 return redirect( builds ) 454 build = Build.objects.get( pk = build_id ); 455 layerVersionId = Layer_Version.objects.filter( build = build_id ); 456 recipeCount = Recipe.objects.filter( layer_version__id__in = layerVersionId ).count( ); 457 tgts = Target.objects.filter( build_id = build_id ).order_by( 'target' ); 458 459 # set up custom target list with computed package and image data 460 targets = [] 461 ntargets = 0 462 463 # True if at least one target for this build has an SDK artifact 464 # or image file 465 has_artifacts = False 466 467 for t in tgts: 468 elem = {} 469 elem['target'] = t 470 471 target_has_images = False 472 image_files = [] 473 474 npkg = 0 475 pkgsz = 0 476 package = None 477 # Chunk the query to avoid "too many SQL variables" error 478 package_set = t.target_installed_package_set.all() 479 package_set_len = len(package_set) 480 for ps_start in range(0,package_set_len,500): 481 ps_stop = min(ps_start+500,package_set_len) 482 for package in Package.objects.filter(id__in = [x.package_id for x in package_set[ps_start:ps_stop]]): 483 pkgsz = pkgsz + package.size 484 if package.installed_name: 485 npkg = npkg + 1 486 elem['npkg'] = npkg 487 elem['pkgsz'] = pkgsz 488 ti = Target_Image_File.objects.filter(target_id = t.id) 489 for i in ti: 490 ndx = i.file_name.rfind('/') 491 if ndx < 0: 492 ndx = 0; 493 f = i.file_name[ndx + 1:] 494 image_files.append({ 495 'id': i.id, 496 'path': f, 497 'size': i.file_size, 498 'suffix': i.suffix 499 }) 500 if len(image_files) > 0: 501 target_has_images = True 502 elem['targetHasImages'] = target_has_images 503 504 elem['imageFiles'] = image_files 505 elem['target_kernel_artifacts'] = t.targetkernelfile_set.all() 506 507 target_sdk_files = t.targetsdkfile_set.all() 508 target_sdk_artifacts_count = target_sdk_files.count() 509 elem['target_sdk_artifacts_count'] = target_sdk_artifacts_count 510 elem['target_sdk_artifacts'] = target_sdk_files 511 512 if target_has_images or target_sdk_artifacts_count > 0: 513 has_artifacts = True 514 515 targets.append(elem) 516 517 ## 518 # how many packages in this build - ignore anonymous ones 519 # 520 521 packageCount = 0 522 packages = Package.objects.filter( build_id = build_id ) 523 for p in packages: 524 if ( p.installed_name ): 525 packageCount = packageCount + 1 526 527 logmessages = list(LogMessage.objects.filter( build = build_id )) 528 529 context = { 530 'build' : build, 531 'project' : build.project, 532 'hasArtifacts' : has_artifacts, 533 'ntargets' : ntargets, 534 'targets' : targets, 535 'recipecount' : recipeCount, 536 'packagecount' : packageCount, 537 'logmessages' : logmessages, 538 } 539 return toaster_render( request, template, context ) 540 541 542 543def generateCoveredList2( revlist = None ): 544 if not revlist: 545 revlist = [] 546 covered_list = [ x for x in revlist if x.outcome == Task.OUTCOME_COVERED ] 547 while len(covered_list): 548 revlist = [ x for x in revlist if x.outcome != Task.OUTCOME_COVERED ] 549 if len(revlist) > 0: 550 return revlist 551 552 newlist = _find_task_revdep_list(covered_list) 553 554 revlist = list(set(revlist + newlist)) 555 covered_list = [ x for x in revlist if x.outcome == Task.OUTCOME_COVERED ] 556 return revlist 557 558def task( request, build_id, task_id ): 559 template = "task.html" 560 tasks_list = Task.objects.filter( pk=task_id ) 561 if tasks_list.count( ) == 0: 562 return redirect( builds ) 563 task_object = tasks_list[ 0 ]; 564 dependencies = sorted( 565 _find_task_dep( task_object ), 566 key=lambda t:'%s_%s %s'%(t.recipe.name, t.recipe.version, t.task_name)) 567 reverse_dependencies = sorted( 568 _find_task_revdep( task_object ), 569 key=lambda t:'%s_%s %s'%( t.recipe.name, t.recipe.version, t.task_name )) 570 coveredBy = ''; 571 if ( task_object.outcome == Task.OUTCOME_COVERED ): 572# _list = generateCoveredList( task ) 573 coveredBy = sorted(generateCoveredList2( _find_task_revdep( task_object ) ), key = lambda x: x.recipe.name) 574 log_head = '' 575 log_body = '' 576 if task_object.outcome == task_object.OUTCOME_FAILED: 577 pass 578 579 uri_list= [ ] 580 variables = Variable.objects.filter(build=build_id) 581 v=variables.filter(variable_name='SSTATE_DIR') 582 if v.count() > 0: 583 uri_list.append(v[0].variable_value) 584 v=variables.filter(variable_name='SSTATE_MIRRORS') 585 if (v.count() > 0): 586 for mirror in v[0].variable_value.split('\\n'): 587 s=re.sub('.* ','',mirror.strip(' \t\n\r')) 588 if len(s): 589 uri_list.append(s) 590 591 context = { 592 'build' : Build.objects.filter( pk = build_id )[ 0 ], 593 'object' : task_object, 594 'task' : task_object, 595 'covered_by' : coveredBy, 596 'deps' : dependencies, 597 'rdeps' : reverse_dependencies, 598 'log_head' : log_head, 599 'log_body' : log_body, 600 'showing_matches' : False, 601 'uri_list' : uri_list, 602 'task_in_tasks_table_pg': int(task_object.order / 25) + 1 603 } 604 if request.GET.get( 'show_matches', "" ): 605 context[ 'showing_matches' ] = True 606 context[ 'matching_tasks' ] = Task.objects.filter( 607 sstate_checksum=task_object.sstate_checksum ).filter( 608 build__completed_on__lt=task_object.build.completed_on).exclude( 609 order__isnull=True).exclude(outcome=Task.OUTCOME_NA).order_by('-build__completed_on') 610 611 return toaster_render( request, template, context ) 612 613def recipe(request, build_id, recipe_id, active_tab="1"): 614 template = "recipe.html" 615 if Recipe.objects.filter(pk=recipe_id).count() == 0 : 616 return redirect(builds) 617 618 recipe_object = Recipe.objects.get(pk=recipe_id) 619 layer_version = Layer_Version.objects.get(pk=recipe_object.layer_version_id) 620 layer = Layer.objects.get(pk=layer_version.layer_id) 621 tasks_list = Task.objects.filter(recipe_id = recipe_id, build_id = build_id).exclude(order__isnull=True).exclude(task_name__endswith='_setscene').exclude(outcome=Task.OUTCOME_NA) 622 package_count = Package.objects.filter(recipe_id = recipe_id).filter(build_id = build_id).filter(size__gte=0).count() 623 624 if active_tab != '1' and active_tab != '3' and active_tab != '4' : 625 active_tab = '1' 626 tab_states = {'1': '', '3': '', '4': ''} 627 tab_states[active_tab] = 'active' 628 629 context = { 630 'build' : Build.objects.get(pk=build_id), 631 'object' : recipe_object, 632 'layer_version' : layer_version, 633 'layer' : layer, 634 'tasks' : tasks_list, 635 'package_count' : package_count, 636 'tab_states' : tab_states, 637 } 638 return toaster_render(request, template, context) 639 640def recipe_packages(request, build_id, recipe_id): 641 template = "recipe_packages.html" 642 if Recipe.objects.filter(pk=recipe_id).count() == 0 : 643 return redirect(builds) 644 645 (pagesize, orderby) = _get_parameters_values(request, 10, 'name:+') 646 mandatory_parameters = { 'count': pagesize, 'page' : 1, 'orderby': orderby } 647 retval = _verify_parameters( request.GET, mandatory_parameters ) 648 if retval: 649 return _redirect_parameters( 'recipe_packages', request.GET, mandatory_parameters, build_id = build_id, recipe_id = recipe_id) 650 (filter_string, search_term, ordering_string) = _search_tuple(request, Package) 651 652 recipe_object = Recipe.objects.get(pk=recipe_id) 653 queryset = Package.objects.filter(recipe_id = recipe_id).filter(build_id = build_id).filter(size__gte=0) 654 package_count = queryset.count() 655 queryset = _get_queryset(Package, queryset, filter_string, search_term, ordering_string, 'name') 656 657 packages = _build_page_range(Paginator(queryset, pagesize),request.GET.get('page', 1)) 658 659 context = { 660 'build' : Build.objects.get(pk=build_id), 661 'recipe' : recipe_object, 662 'objects' : packages, 663 'object_count' : package_count, 664 'tablecols':[ 665 { 666 'name':'Package', 667 'orderfield': _get_toggle_order(request,"name"), 668 'ordericon': _get_toggle_order_icon(request,"name"), 669 'orderkey': "name", 670 }, 671 { 672 'name':'Version', 673 }, 674 { 675 'name':'Size', 676 'orderfield': _get_toggle_order(request,"size", True), 677 'ordericon': _get_toggle_order_icon(request,"size"), 678 'orderkey': 'size', 679 'dclass': 'sizecol span2', 680 }, 681 ] 682 } 683 response = toaster_render(request, template, context) 684 _set_parameters_values(pagesize, orderby, request) 685 return response 686 687from django.http import HttpResponse 688@log_view_mixin 689def xhr_dirinfo(request, build_id, target_id): 690 top = request.GET.get('start', '/') 691 return HttpResponse(_get_dir_entries(build_id, target_id, top), content_type = "application/json") 692 693from django.utils.functional import Promise 694from django.utils.encoding import force_str 695class LazyEncoder(json.JSONEncoder): 696 def default(self, obj): 697 if isinstance(obj, Promise): 698 return force_str(obj) 699 return super(LazyEncoder, self).default(obj) 700 701from toastergui.templatetags.projecttags import filtered_filesizeformat 702import os 703def _get_dir_entries(build_id, target_id, start): 704 node_str = { 705 Target_File.ITYPE_REGULAR : '-', 706 Target_File.ITYPE_DIRECTORY : 'd', 707 Target_File.ITYPE_SYMLINK : 'l', 708 Target_File.ITYPE_SOCKET : 's', 709 Target_File.ITYPE_FIFO : 'p', 710 Target_File.ITYPE_CHARACTER : 'c', 711 Target_File.ITYPE_BLOCK : 'b', 712 } 713 response = [] 714 objects = Target_File.objects.filter(target__exact=target_id, directory__path=start) 715 target_packages = Target_Installed_Package.objects.filter(target__exact=target_id).values_list('package_id', flat=True) 716 for o in objects: 717 # exclude root inode '/' 718 if o.path == '/': 719 continue 720 try: 721 entry = {} 722 entry['parent'] = start 723 entry['name'] = os.path.basename(o.path) 724 entry['fullpath'] = o.path 725 726 # set defaults, not all dentries have packages 727 entry['installed_package'] = None 728 entry['package_id'] = None 729 entry['package'] = None 730 entry['link_to'] = None 731 if o.inodetype == Target_File.ITYPE_DIRECTORY: 732 entry['isdir'] = 1 733 # is there content in directory 734 entry['childcount'] = Target_File.objects.filter(target__exact=target_id, directory__path=o.path).all().count() 735 else: 736 entry['isdir'] = 0 737 738 # resolve the file to get the package from the resolved file 739 resolved_id = o.sym_target_id 740 resolved_path = o.path 741 if target_packages.count(): 742 while resolved_id != "" and resolved_id is not None: 743 tf = Target_File.objects.get(pk=resolved_id) 744 resolved_path = tf.path 745 resolved_id = tf.sym_target_id 746 747 thisfile=Package_File.objects.all().filter(path__exact=resolved_path, package_id__in=target_packages) 748 if thisfile.count(): 749 p = Package.objects.get(pk=thisfile[0].package_id) 750 entry['installed_package'] = p.installed_name 751 entry['package_id'] = str(p.id) 752 entry['package'] = p.name 753 # don't use resolved path from above, show immediate link-to 754 if o.sym_target_id != "" and o.sym_target_id is not None: 755 entry['link_to'] = Target_File.objects.get(pk=o.sym_target_id).path 756 entry['size'] = filtered_filesizeformat(o.size) 757 if entry['link_to'] is not None: 758 entry['permission'] = node_str[o.inodetype] + o.permission 759 else: 760 entry['permission'] = node_str[o.inodetype] + o.permission 761 entry['owner'] = o.owner 762 entry['group'] = o.group 763 response.append(entry) 764 765 except Exception as e: 766 print("Exception ", e) 767 traceback.print_exc() 768 769 # sort by directories first, then by name 770 rsorted = sorted(response, key=lambda entry : entry['name']) 771 rsorted = sorted(rsorted, key=lambda entry : entry['isdir'], reverse=True) 772 return json.dumps(rsorted, cls=LazyEncoder).replace('</', '<\\/') 773 774def dirinfo(request, build_id, target_id, file_path=None): 775 template = "dirinfo.html" 776 objects = _get_dir_entries(build_id, target_id, '/') 777 packages_sum = Package.objects.filter(id__in=Target_Installed_Package.objects.filter(target_id=target_id).values('package_id')).aggregate(Sum('installed_size')) 778 dir_list = None 779 if file_path is not None: 780 """ 781 Link from the included package detail file list page and is 782 requesting opening the dir info to a specific file path. 783 Provide the list of directories to expand and the full path to 784 highlight in the page. 785 """ 786 # Aassume target's path separator matches host's, that is, os.sep 787 sep = os.sep 788 dir_list = [] 789 head = file_path 790 while head != sep: 791 (head, tail) = os.path.split(head) 792 if head != sep: 793 dir_list.insert(0, head) 794 795 build = Build.objects.get(pk=build_id) 796 797 context = { 'build': build, 798 'project': build.project, 799 'target': Target.objects.get(pk=target_id), 800 'packages_sum': packages_sum['installed_size__sum'], 801 'objects': objects, 802 'dir_list': dir_list, 803 'file_path': file_path, 804 } 805 return toaster_render(request, template, context) 806 807def _find_task_dep(task_object): 808 tdeps = Task_Dependency.objects.filter(task=task_object).filter(depends_on__order__gt=0) 809 tdeps = tdeps.exclude(depends_on__outcome=Task.OUTCOME_NA).select_related("depends_on") 810 return [x.depends_on for x in tdeps] 811 812def _find_task_revdep(task_object): 813 tdeps = Task_Dependency.objects.filter(depends_on=task_object).filter(task__order__gt=0) 814 tdeps = tdeps.exclude(task__outcome = Task.OUTCOME_NA).select_related("task", "task__recipe", "task__build") 815 816 # exclude self-dependencies to prevent infinite dependency loop 817 # in generateCoveredList2() 818 tdeps = tdeps.exclude(task=task_object) 819 820 return [tdep.task for tdep in tdeps] 821 822def _find_task_revdep_list(tasklist): 823 tdeps = Task_Dependency.objects.filter(depends_on__in=tasklist).filter(task__order__gt=0) 824 tdeps = tdeps.exclude(task__outcome=Task.OUTCOME_NA).select_related("task", "task__recipe", "task__build") 825 826 # exclude self-dependencies to prevent infinite dependency loop 827 # in generateCoveredList2() 828 tdeps = tdeps.exclude(task=F('depends_on')) 829 830 return [tdep.task for tdep in tdeps] 831 832def _find_task_provider(task_object): 833 task_revdeps = _find_task_revdep(task_object) 834 for tr in task_revdeps: 835 if tr.outcome != Task.OUTCOME_COVERED: 836 return tr 837 for tr in task_revdeps: 838 trc = _find_task_provider(tr) 839 if trc is not None: 840 return trc 841 return None 842 843def configuration(request, build_id): 844 template = 'configuration.html' 845 846 var_names = ('BB_VERSION', 'BUILD_SYS', 'NATIVELSBSTRING', 'TARGET_SYS', 847 'MACHINE', 'DISTRO', 'DISTRO_VERSION', 'TUNE_FEATURES', 'TARGET_FPU') 848 context = dict(Variable.objects.filter(build=build_id, variable_name__in=var_names)\ 849 .values_list('variable_name', 'variable_value')) 850 build = Build.objects.get(pk=build_id) 851 context.update({'objectname': 'configuration', 852 'object_search_display':'variables', 853 'filter_search_display':'variables', 854 'build': build, 855 'project': build.project, 856 'targets': Target.objects.filter(build=build_id)}) 857 return toaster_render(request, template, context) 858 859 860def configvars(request, build_id): 861 template = 'configvars.html' 862 (pagesize, orderby) = _get_parameters_values(request, 100, 'variable_name:+') 863 mandatory_parameters = { 'count': pagesize, 'page' : 1, 'orderby' : orderby, 'filter' : 'description__regex:.+' } 864 retval = _verify_parameters( request.GET, mandatory_parameters ) 865 (filter_string, search_term, ordering_string) = _search_tuple(request, Variable) 866 if retval: 867 # if new search, clear the default filter 868 if search_term and len(search_term): 869 mandatory_parameters['filter']='' 870 return _redirect_parameters( 'configvars', request.GET, mandatory_parameters, build_id = build_id) 871 872 queryset = Variable.objects.filter(build=build_id).exclude(variable_name__istartswith='B_').exclude(variable_name__istartswith='do_') 873 queryset_with_search = _get_queryset(Variable, queryset, None, search_term, ordering_string, 'variable_name').exclude(variable_value='',vhistory__file_name__isnull=True) 874 queryset = _get_queryset(Variable, queryset, filter_string, search_term, ordering_string, 'variable_name') 875 # remove records where the value is empty AND there are no history files 876 queryset = queryset.exclude(variable_value='',vhistory__file_name__isnull=True) 877 878 variables = _build_page_range(Paginator(queryset, pagesize), request.GET.get('page', 1)) 879 880 # show all matching files (not just the last one) 881 file_filter= search_term + ":" 882 if filter_string.find('/conf/') > 0: 883 file_filter += 'conf/(local|bblayers).conf' 884 if filter_string.find('conf/machine/') > 0: 885 file_filter += 'conf/machine/' 886 if filter_string.find('conf/distro/') > 0: 887 file_filter += 'conf/distro/' 888 if filter_string.find('/bitbake.conf') > 0: 889 file_filter += '/bitbake.conf' 890 build_dir=re.sub("/tmp/log/.*","",Build.objects.get(pk=build_id).cooker_log_path) 891 892 build = Build.objects.get(pk=build_id) 893 894 context = { 895 'objectname': 'configvars', 896 'object_search_display':'BitBake variables', 897 'filter_search_display':'variables', 898 'file_filter': file_filter, 899 'build': build, 900 'project': build.project, 901 'objects' : variables, 902 'total_count':queryset_with_search.count(), 903 'default_orderby' : 'variable_name:+', 904 'search_term':search_term, 905 # Specifies the display of columns for the table, appearance in "Edit columns" box, toggling default show/hide, and specifying filters for columns 906 'tablecols' : [ 907 {'name': 'Variable', 908 'qhelp': "BitBake is a generic task executor that considers a list of tasks with dependencies and handles metadata that consists of variables in a certain format that get passed to the tasks", 909 'orderfield': _get_toggle_order(request, "variable_name"), 910 'ordericon':_get_toggle_order_icon(request, "variable_name"), 911 }, 912 {'name': 'Value', 913 'qhelp': "The value assigned to the variable", 914 }, 915 {'name': 'Set in file', 916 'qhelp': "The last configuration file that touched the variable value", 917 'clclass': 'file', 'hidden' : 0, 918 'orderkey' : 'vhistory__file_name', 919 'filter' : { 920 'class' : 'vhistory__file_name', 921 'label': 'Show:', 922 'options' : [ 923 ('Local configuration variables', 'vhistory__file_name__contains:'+build_dir+'/conf/',queryset_with_search.filter(vhistory__file_name__contains=build_dir+'/conf/').count(), 'Select this filter to see variables set by the <code>local.conf</code> and <code>bblayers.conf</code> configuration files inside the <code>/build/conf/</code> directory'), 924 ('Machine configuration variables', 'vhistory__file_name__contains:conf/machine/',queryset_with_search.filter(vhistory__file_name__contains='conf/machine').count(), 'Select this filter to see variables set by the configuration file(s) inside your layers <code>/conf/machine/</code> directory'), 925 ('Distro configuration variables', 'vhistory__file_name__contains:conf/distro/',queryset_with_search.filter(vhistory__file_name__contains='conf/distro').count(), 'Select this filter to see variables set by the configuration file(s) inside your layers <code>/conf/distro/</code> directory'), 926 ('Layer configuration variables', 'vhistory__file_name__contains:conf/layer.conf',queryset_with_search.filter(vhistory__file_name__contains='conf/layer.conf').count(), 'Select this filter to see variables set by the <code>layer.conf</code> configuration file inside your layers'), 927 ('bitbake.conf variables', 'vhistory__file_name__contains:/bitbake.conf',queryset_with_search.filter(vhistory__file_name__contains='/bitbake.conf').count(), 'Select this filter to see variables set by the <code>bitbake.conf</code> configuration file'), 928 ] 929 }, 930 }, 931 {'name': 'Description', 932 'qhelp': "A brief explanation of the variable", 933 'clclass': 'description', 'hidden' : 0, 934 'dclass': "span4", 935 'filter' : { 936 'class' : 'description', 937 'label': 'Show:', 938 'options' : [ 939 ('Variables with description', 'description__regex:.+', queryset_with_search.filter(description__regex='.+').count(), 'We provide descriptions for the most common BitBake variables. The list of descriptions lives in <code>meta/conf/documentation.conf</code>'), 940 ] 941 }, 942 }, 943 ], 944 } 945 946 response = toaster_render(request, template, context) 947 _set_parameters_values(pagesize, orderby, request) 948 return response 949 950def bfile(request, build_id, package_id): 951 template = 'bfile.html' 952 files = Package_File.objects.filter(package = package_id) 953 build = Build.objects.get(pk=build_id) 954 context = { 955 'build': build, 956 'project': build.project, 957 'objects' : files 958 } 959 return toaster_render(request, template, context) 960 961 962# A set of dependency types valid for both included and built package views 963OTHER_DEPENDS_BASE = [ 964 Package_Dependency.TYPE_RSUGGESTS, 965 Package_Dependency.TYPE_RPROVIDES, 966 Package_Dependency.TYPE_RREPLACES, 967 Package_Dependency.TYPE_RCONFLICTS, 968 ] 969 970# value for invalid row id 971INVALID_KEY = -1 972 973""" 974Given a package id, target_id retrieves two sets of this image and package's 975dependencies. The return value is a dictionary consisting of two other 976lists: a list of 'runtime' dependencies, that is, having RDEPENDS 977values in source package's recipe, and a list of other dependencies, that is 978the list of possible recipe variables as found in OTHER_DEPENDS_BASE plus 979the RRECOMMENDS or TRECOMMENDS value. 980The lists are built in the sort order specified for the package runtime 981dependency views. 982""" 983def _get_package_dependencies(package_id, target_id = INVALID_KEY): 984 runtime_deps = [] 985 other_deps = [] 986 other_depends_types = OTHER_DEPENDS_BASE 987 988 if target_id != INVALID_KEY : 989 rdepends_type = Package_Dependency.TYPE_TRDEPENDS 990 other_depends_types += [Package_Dependency.TYPE_TRECOMMENDS] 991 else : 992 rdepends_type = Package_Dependency.TYPE_RDEPENDS 993 other_depends_types += [Package_Dependency.TYPE_RRECOMMENDS] 994 995 package = Package.objects.get(pk=package_id) 996 if target_id != INVALID_KEY : 997 alldeps = package.package_dependencies_source.filter(target_id__exact = target_id) 998 else : 999 alldeps = package.package_dependencies_source.all() 1000 for idep in alldeps: 1001 dep_package = Package.objects.get(pk=idep.depends_on_id) 1002 dep_entry = Package_Dependency.DEPENDS_DICT[idep.dep_type] 1003 if dep_package.version == '' : 1004 version = '' 1005 else : 1006 version = dep_package.version + "-" + dep_package.revision 1007 installed = False 1008 if target_id != INVALID_KEY : 1009 if Target_Installed_Package.objects.filter(target_id__exact = target_id, package_id__exact = dep_package.id).count() > 0: 1010 installed = True 1011 dep = { 1012 'name' : dep_package.name, 1013 'version' : version, 1014 'size' : dep_package.size, 1015 'dep_type' : idep.dep_type, 1016 'dep_type_display' : dep_entry[0].capitalize(), 1017 'dep_type_help' : dep_entry[1] % (dep_package.name, package.name), 1018 'depends_on_id' : dep_package.id, 1019 'installed' : installed, 1020 } 1021 1022 if target_id != INVALID_KEY: 1023 dep['alias'] = _get_package_alias(dep_package) 1024 1025 if idep.dep_type == rdepends_type : 1026 runtime_deps.append(dep) 1027 elif idep.dep_type in other_depends_types : 1028 other_deps.append(dep) 1029 1030 rdep_sorted = sorted(runtime_deps, key=lambda k: k['name']) 1031 odep_sorted = sorted( 1032 sorted(other_deps, key=lambda k: k['name']), 1033 key=lambda k: k['dep_type']) 1034 retvalues = {'runtime_deps' : rdep_sorted, 'other_deps' : odep_sorted} 1035 return retvalues 1036 1037# Return the count of packages dependent on package for this target_id image 1038def _get_package_reverse_dep_count(package, target_id): 1039 return package.package_dependencies_target.filter(target_id__exact=target_id, dep_type__exact = Package_Dependency.TYPE_TRDEPENDS).count() 1040 1041# Return the count of the packages that this package_id is dependent on. 1042# Use one of the two RDEPENDS types, either TRDEPENDS if the package was 1043# installed, or else RDEPENDS if only built. 1044def _get_package_dependency_count(package, target_id, is_installed): 1045 if is_installed : 1046 return package.package_dependencies_source.filter(target_id__exact = target_id, 1047 dep_type__exact = Package_Dependency.TYPE_TRDEPENDS).count() 1048 else : 1049 return package.package_dependencies_source.filter(dep_type__exact = Package_Dependency.TYPE_RDEPENDS).count() 1050 1051def _get_package_alias(package): 1052 alias = package.installed_name 1053 if alias is not None and alias != '' and alias != package.name: 1054 return alias 1055 else: 1056 return '' 1057 1058def _get_fullpackagespec(package): 1059 r = package.name 1060 version_good = package.version is not None and package.version != '' 1061 revision_good = package.revision is not None and package.revision != '' 1062 if version_good or revision_good: 1063 r += '_' 1064 if version_good: 1065 r += package.version 1066 if revision_good: 1067 r += '-' 1068 if revision_good: 1069 r += package.revision 1070 return r 1071 1072def package_built_detail(request, build_id, package_id): 1073 template = "package_built_detail.html" 1074 if Build.objects.filter(pk=build_id).count() == 0 : 1075 return redirect(builds) 1076 1077 # follow convention for pagination w/ search although not used for this view 1078 queryset = Package_File.objects.filter(package_id__exact=package_id) 1079 (pagesize, orderby) = _get_parameters_values(request, 25, 'path:+') 1080 mandatory_parameters = { 'count': pagesize, 'page' : 1, 'orderby' : orderby } 1081 retval = _verify_parameters( request.GET, mandatory_parameters ) 1082 if retval: 1083 return _redirect_parameters( 'package_built_detail', request.GET, mandatory_parameters, build_id = build_id, package_id = package_id) 1084 1085 (filter_string, search_term, ordering_string) = _search_tuple(request, Package_File) 1086 paths = _get_queryset(Package_File, queryset, filter_string, search_term, ordering_string, 'path') 1087 1088 package = Package.objects.get(pk=package_id) 1089 package.fullpackagespec = _get_fullpackagespec(package) 1090 context = { 1091 'build' : Build.objects.get(pk=build_id), 1092 'package' : package, 1093 'dependency_count' : _get_package_dependency_count(package, -1, False), 1094 'objects' : paths, 1095 'tablecols':[ 1096 { 1097 'name':'File', 1098 'orderfield': _get_toggle_order(request, "path"), 1099 'ordericon':_get_toggle_order_icon(request, "path"), 1100 }, 1101 { 1102 'name':'Size', 1103 'orderfield': _get_toggle_order(request, "size", True), 1104 'ordericon':_get_toggle_order_icon(request, "size"), 1105 'dclass': 'sizecol span2', 1106 }, 1107 ] 1108 } 1109 if paths.all().count() < 2: 1110 context['disable_sort'] = True; 1111 1112 response = toaster_render(request, template, context) 1113 _set_parameters_values(pagesize, orderby, request) 1114 return response 1115 1116def package_built_dependencies(request, build_id, package_id): 1117 template = "package_built_dependencies.html" 1118 if Build.objects.filter(pk=build_id).count() == 0 : 1119 return redirect(builds) 1120 1121 package = Package.objects.get(pk=package_id) 1122 package.fullpackagespec = _get_fullpackagespec(package) 1123 dependencies = _get_package_dependencies(package_id) 1124 context = { 1125 'build' : Build.objects.get(pk=build_id), 1126 'package' : package, 1127 'runtime_deps' : dependencies['runtime_deps'], 1128 'other_deps' : dependencies['other_deps'], 1129 'dependency_count' : _get_package_dependency_count(package, -1, False) 1130 } 1131 return toaster_render(request, template, context) 1132 1133 1134def package_included_detail(request, build_id, target_id, package_id): 1135 template = "package_included_detail.html" 1136 if Build.objects.filter(pk=build_id).count() == 0 : 1137 return redirect(builds) 1138 1139 # follow convention for pagination w/ search although not used for this view 1140 (pagesize, orderby) = _get_parameters_values(request, 25, 'path:+') 1141 mandatory_parameters = { 'count': pagesize, 'page' : 1, 'orderby' : orderby } 1142 retval = _verify_parameters( request.GET, mandatory_parameters ) 1143 if retval: 1144 return _redirect_parameters( 'package_included_detail', request.GET, mandatory_parameters, build_id = build_id, target_id = target_id, package_id = package_id) 1145 (filter_string, search_term, ordering_string) = _search_tuple(request, Package_File) 1146 1147 queryset = Package_File.objects.filter(package_id__exact=package_id) 1148 paths = _get_queryset(Package_File, queryset, filter_string, search_term, ordering_string, 'path') 1149 1150 package = Package.objects.get(pk=package_id) 1151 package.fullpackagespec = _get_fullpackagespec(package) 1152 package.alias = _get_package_alias(package) 1153 target = Target.objects.get(pk=target_id) 1154 context = { 1155 'build' : Build.objects.get(pk=build_id), 1156 'target' : target, 1157 'package' : package, 1158 'reverse_count' : _get_package_reverse_dep_count(package, target_id), 1159 'dependency_count' : _get_package_dependency_count(package, target_id, True), 1160 'objects': paths, 1161 'tablecols':[ 1162 { 1163 'name':'File', 1164 'orderfield': _get_toggle_order(request, "path"), 1165 'ordericon':_get_toggle_order_icon(request, "path"), 1166 }, 1167 { 1168 'name':'Size', 1169 'orderfield': _get_toggle_order(request, "size", True), 1170 'ordericon':_get_toggle_order_icon(request, "size"), 1171 'dclass': 'sizecol span2', 1172 }, 1173 ] 1174 } 1175 if paths.all().count() < 2: 1176 context['disable_sort'] = True 1177 response = toaster_render(request, template, context) 1178 _set_parameters_values(pagesize, orderby, request) 1179 return response 1180 1181def package_included_dependencies(request, build_id, target_id, package_id): 1182 template = "package_included_dependencies.html" 1183 if Build.objects.filter(pk=build_id).count() == 0 : 1184 return redirect(builds) 1185 1186 package = Package.objects.get(pk=package_id) 1187 package.fullpackagespec = _get_fullpackagespec(package) 1188 package.alias = _get_package_alias(package) 1189 target = Target.objects.get(pk=target_id) 1190 1191 dependencies = _get_package_dependencies(package_id, target_id) 1192 context = { 1193 'build' : Build.objects.get(pk=build_id), 1194 'package' : package, 1195 'target' : target, 1196 'runtime_deps' : dependencies['runtime_deps'], 1197 'other_deps' : dependencies['other_deps'], 1198 'reverse_count' : _get_package_reverse_dep_count(package, target_id), 1199 'dependency_count' : _get_package_dependency_count(package, target_id, True) 1200 } 1201 return toaster_render(request, template, context) 1202 1203def package_included_reverse_dependencies(request, build_id, target_id, package_id): 1204 template = "package_included_reverse_dependencies.html" 1205 if Build.objects.filter(pk=build_id).count() == 0 : 1206 return redirect(builds) 1207 1208 (pagesize, orderby) = _get_parameters_values(request, 25, 'package__name:+') 1209 mandatory_parameters = { 'count': pagesize, 'page' : 1, 'orderby': orderby } 1210 retval = _verify_parameters( request.GET, mandatory_parameters ) 1211 if retval: 1212 return _redirect_parameters( 'package_included_reverse_dependencies', request.GET, mandatory_parameters, build_id = build_id, target_id = target_id, package_id = package_id) 1213 (filter_string, search_term, ordering_string) = _search_tuple(request, Package_File) 1214 1215 queryset = Package_Dependency.objects.select_related('depends_on').filter(depends_on=package_id, target_id=target_id, dep_type=Package_Dependency.TYPE_TRDEPENDS) 1216 objects = _get_queryset(Package_Dependency, queryset, filter_string, search_term, ordering_string, 'package__name') 1217 1218 package = Package.objects.get(pk=package_id) 1219 package.fullpackagespec = _get_fullpackagespec(package) 1220 package.alias = _get_package_alias(package) 1221 target = Target.objects.get(pk=target_id) 1222 for o in objects: 1223 if o.package.version != '': 1224 o.package.version += '-' + o.package.revision 1225 o.alias = _get_package_alias(o.package) 1226 context = { 1227 'build' : Build.objects.get(pk=build_id), 1228 'package' : package, 1229 'target' : target, 1230 'objects' : objects, 1231 'reverse_count' : _get_package_reverse_dep_count(package, target_id), 1232 'dependency_count' : _get_package_dependency_count(package, target_id, True), 1233 'tablecols':[ 1234 { 1235 'name':'Package', 1236 'orderfield': _get_toggle_order(request, "package__name"), 1237 'ordericon': _get_toggle_order_icon(request, "package__name"), 1238 }, 1239 { 1240 'name':'Version', 1241 }, 1242 { 1243 'name':'Size', 1244 'orderfield': _get_toggle_order(request, "package__size", True), 1245 'ordericon': _get_toggle_order_icon(request, "package__size"), 1246 'dclass': 'sizecol span2', 1247 }, 1248 ] 1249 } 1250 if objects.all().count() < 2: 1251 context['disable_sort'] = True 1252 response = toaster_render(request, template, context) 1253 _set_parameters_values(pagesize, orderby, request) 1254 return response 1255 1256def image_information_dir(request, build_id, target_id, packagefile_id): 1257 # stubbed for now 1258 return redirect(builds) 1259 # the context processor that supplies data used across all the pages 1260 1261# a context processor which runs on every request; this provides the 1262# projects and non_cli_projects (i.e. projects created by the user) 1263# variables referred to in templates, which used to determine the 1264# visibility of UI elements like the "New build" button 1265def managedcontextprocessor(request): 1266 projects = Project.objects.all() 1267 ret = { 1268 "projects": projects, 1269 "non_cli_projects": projects.exclude(is_default=True), 1270 "DEBUG" : toastermain.settings.DEBUG, 1271 "TOASTER_BRANCH": toastermain.settings.TOASTER_BRANCH, 1272 "TOASTER_REVISION" : toastermain.settings.TOASTER_REVISION, 1273 } 1274 return ret 1275 1276# REST-based API calls to return build/building status to external Toaster 1277# managers and aggregators via JSON 1278 1279def _json_build_status(build_id,extend): 1280 build_stat = None 1281 try: 1282 build = Build.objects.get( pk = build_id ) 1283 build_stat = {} 1284 build_stat['id'] = build.id 1285 build_stat['name'] = build.build_name 1286 build_stat['machine'] = build.machine 1287 build_stat['distro'] = build.distro 1288 build_stat['start'] = build.started_on 1289 # look up target name 1290 target= Target.objects.get( build = build ) 1291 if target: 1292 if target.task: 1293 build_stat['target'] = '%s:%s' % (target.target,target.task) 1294 else: 1295 build_stat['target'] = '%s' % (target.target) 1296 else: 1297 build_stat['target'] = '' 1298 # look up project name 1299 project = Project.objects.get( build = build ) 1300 if project: 1301 build_stat['project'] = project.name 1302 else: 1303 build_stat['project'] = '' 1304 if Build.IN_PROGRESS == build.outcome: 1305 now = timezone.now() 1306 timediff = now - build.started_on 1307 build_stat['seconds']='%.3f' % timediff.total_seconds() 1308 build_stat['clone']='%d:%d' % (build.repos_cloned,build.repos_to_clone) 1309 build_stat['parse']='%d:%d' % (build.recipes_parsed,build.recipes_to_parse) 1310 tf = Task.objects.filter(build = build) 1311 tfc = tf.count() 1312 if tfc > 0: 1313 tfd = tf.exclude(order__isnull=True).count() 1314 else: 1315 tfd = 0 1316 build_stat['task']='%d:%d' % (tfd,tfc) 1317 else: 1318 build_stat['outcome'] = build.get_outcome_text() 1319 timediff = build.completed_on - build.started_on 1320 build_stat['seconds']='%.3f' % timediff.total_seconds() 1321 build_stat['stop'] = build.completed_on 1322 messages = LogMessage.objects.all().filter(build = build) 1323 errors = len(messages.filter(level=LogMessage.ERROR) | 1324 messages.filter(level=LogMessage.EXCEPTION) | 1325 messages.filter(level=LogMessage.CRITICAL)) 1326 build_stat['errors'] = errors 1327 warnings = len(messages.filter(level=LogMessage.WARNING)) 1328 build_stat['warnings'] = warnings 1329 if extend: 1330 build_stat['cooker_log'] = build.cooker_log_path 1331 except Exception as e: 1332 build_state = str(e) 1333 return build_stat 1334 1335def json_builds(request): 1336 build_table = [] 1337 builds = [] 1338 try: 1339 builds = Build.objects.exclude(outcome=Build.IN_PROGRESS).order_by("-started_on") 1340 for build in builds: 1341 build_table.append(_json_build_status(build.id,False)) 1342 except Exception as e: 1343 build_table = str(e) 1344 return JsonResponse({'builds' : build_table, 'count' : len(builds)}) 1345 1346def json_building(request): 1347 build_table = [] 1348 builds = [] 1349 try: 1350 builds = Build.objects.filter(outcome=Build.IN_PROGRESS).order_by("-started_on") 1351 for build in builds: 1352 build_table.append(_json_build_status(build.id,False)) 1353 except Exception as e: 1354 build_table = str(e) 1355 return JsonResponse({'building' : build_table, 'count' : len(builds)}) 1356 1357def json_build(request,build_id): 1358 return JsonResponse({'build' : _json_build_status(build_id,True)}) 1359 1360 1361import toastermain.settings 1362 1363from orm.models import Project, ProjectLayer, ProjectVariable 1364from bldcontrol.models import BuildEnvironment 1365 1366# we have a set of functions if we're in managed mode, or 1367# a default "page not available" simple functions for interactive mode 1368 1369if True: 1370 from django.contrib.auth.models import User 1371 from django.contrib.auth import authenticate, login 1372 1373 from orm.models import LayerSource, ToasterSetting, Release 1374 1375 import traceback 1376 1377 class BadParameterException(Exception): 1378 ''' The exception raised on invalid POST requests ''' 1379 pass 1380 1381 # new project 1382 def newproject(request): 1383 if not project_enable: 1384 return redirect( landing ) 1385 1386 template = "newproject.html" 1387 context = { 1388 'email': request.user.email if request.user.is_authenticated else '', 1389 'username': request.user.username if request.user.is_authenticated else '', 1390 'releases': Release.objects.order_by("description"), 1391 } 1392 1393 try: 1394 context['defaultbranch'] = ToasterSetting.objects.get(name = "DEFAULT_RELEASE").value 1395 except ToasterSetting.DoesNotExist: 1396 pass 1397 1398 if request.method == "GET": 1399 # render new project page 1400 return toaster_render(request, template, context) 1401 elif request.method == "POST": 1402 mandatory_fields = ['projectname', 'ptype'] 1403 try: 1404 ptype = request.POST.get('ptype') 1405 if ptype == "import": 1406 mandatory_fields.append('importdir') 1407 else: 1408 mandatory_fields.append('projectversion') 1409 # make sure we have values for all mandatory_fields 1410 missing = [field for field in mandatory_fields if len(request.POST.get(field, '')) == 0] 1411 if missing: 1412 # set alert for missing fields 1413 raise BadParameterException("Fields missing: %s" % ", ".join(missing)) 1414 1415 if not request.user.is_authenticated: 1416 user = authenticate(username = request.POST.get('username', '_anonuser'), password = 'nopass') 1417 if user is None: 1418 user = User.objects.create_user(username = request.POST.get('username', '_anonuser'), email = request.POST.get('email', ''), password = "nopass") 1419 1420 user = authenticate(username = user.username, password = 'nopass') 1421 login(request, user) 1422 1423 # save the project 1424 if ptype == "import": 1425 if not os.path.isdir('%s/conf' % request.POST['importdir']): 1426 raise BadParameterException("Bad path or missing 'conf' directory (%s)" % request.POST['importdir']) 1427 from django.core import management 1428 management.call_command('buildimport', '--command=import', '--name=%s' % request.POST['projectname'], '--path=%s' % request.POST['importdir']) 1429 prj = Project.objects.get(name = request.POST['projectname']) 1430 prj.merged_attr = True 1431 prj.save() 1432 else: 1433 release = Release.objects.get(pk = request.POST.get('projectversion', None )) 1434 prj = Project.objects.create_project(name = request.POST['projectname'], release = release) 1435 prj.user_id = request.user.pk 1436 if 'mergeattr' == request.POST.get('mergeattr', ''): 1437 prj.merged_attr = True 1438 prj.save() 1439 1440 return redirect(reverse(project, args=(prj.pk,)) + "?notify=new-project") 1441 1442 except (IntegrityError, BadParameterException) as e: 1443 # fill in page with previously submitted values 1444 for field in mandatory_fields: 1445 context.__setitem__(field, request.POST.get(field, "-- missing")) 1446 if isinstance(e, IntegrityError) and "username" in str(e): 1447 context['alert'] = "Your chosen username is already used" 1448 else: 1449 context['alert'] = str(e) 1450 return toaster_render(request, template, context) 1451 1452 raise Exception("Invalid HTTP method for this page") 1453 1454 # new project 1455 def newproject_specific(request, pid): 1456 if not project_enable: 1457 return redirect( landing ) 1458 1459 project = Project.objects.get(pk=pid) 1460 template = "newproject_specific.html" 1461 context = { 1462 'email': request.user.email if request.user.is_authenticated else '', 1463 'username': request.user.username if request.user.is_authenticated else '', 1464 'releases': Release.objects.order_by("description"), 1465 'projectname': project.name, 1466 'project_pk': project.pk, 1467 } 1468 1469 # WORKAROUND: if we already know release, redirect 'newproject_specific' to 'project_specific' 1470 if '1' == project.get_variable('INTERNAL_PROJECT_SPECIFIC_SKIPRELEASE'): 1471 return redirect(reverse(project_specific, args=(project.pk,))) 1472 1473 try: 1474 context['defaultbranch'] = ToasterSetting.objects.get(name = "DEFAULT_RELEASE").value 1475 except ToasterSetting.DoesNotExist: 1476 pass 1477 1478 if request.method == "GET": 1479 # render new project page 1480 return toaster_render(request, template, context) 1481 elif request.method == "POST": 1482 mandatory_fields = ['projectname', 'ptype'] 1483 try: 1484 ptype = request.POST.get('ptype') 1485 if ptype == "build": 1486 mandatory_fields.append('projectversion') 1487 # make sure we have values for all mandatory_fields 1488 missing = [field for field in mandatory_fields if len(request.POST.get(field, '')) == 0] 1489 if missing: 1490 # set alert for missing fields 1491 raise BadParameterException("Fields missing: %s" % ", ".join(missing)) 1492 1493 if not request.user.is_authenticated: 1494 user = authenticate(username = request.POST.get('username', '_anonuser'), password = 'nopass') 1495 if user is None: 1496 user = User.objects.create_user(username = request.POST.get('username', '_anonuser'), email = request.POST.get('email', ''), password = "nopass") 1497 1498 user = authenticate(username = user.username, password = 'nopass') 1499 login(request, user) 1500 1501 # save the project 1502 if ptype == "analysis": 1503 release = None 1504 else: 1505 release = Release.objects.get(pk = request.POST.get('projectversion', None )) 1506 1507 prj = Project.objects.create_project(name = request.POST['projectname'], release = release, existing_project = project) 1508 prj.user_id = request.user.pk 1509 prj.save() 1510 return redirect(reverse(project_specific, args=(prj.pk,)) + "?notify=new-project") 1511 1512 except (IntegrityError, BadParameterException) as e: 1513 # fill in page with previously submitted values 1514 for field in mandatory_fields: 1515 context.__setitem__(field, request.POST.get(field, "-- missing")) 1516 if isinstance(e, IntegrityError) and "username" in str(e): 1517 context['alert'] = "Your chosen username is already used" 1518 else: 1519 context['alert'] = str(e) 1520 return toaster_render(request, template, context) 1521 1522 raise Exception("Invalid HTTP method for this page") 1523 1524 # Shows the edit project page 1525 def project(request, pid): 1526 project = Project.objects.get(pk=pid) 1527 1528 if '1' == os.environ.get('TOASTER_PROJECTSPECIFIC'): 1529 if request.GET: 1530 #Example:request.GET=<QueryDict: {'setMachine': ['qemuarm']}> 1531 params = urlencode(request.GET).replace('%5B%27','').replace('%27%5D','') 1532 return redirect("%s?%s" % (reverse(project_specific, args=(project.pk,)),params)) 1533 else: 1534 return redirect(reverse(project_specific, args=(project.pk,))) 1535 context = {"project": project} 1536 return toaster_render(request, "project.html", context) 1537 1538 # Shows the edit project-specific page 1539 def project_specific(request, pid): 1540 project = Project.objects.get(pk=pid) 1541 1542 # Are we refreshing from a successful project specific update clone? 1543 if Project.PROJECT_SPECIFIC_CLONING_SUCCESS == project.get_variable(Project.PROJECT_SPECIFIC_STATUS): 1544 return redirect(reverse(landing_specific,args=(project.pk,))) 1545 1546 context = { 1547 "project": project, 1548 "is_new" : project.get_variable(Project.PROJECT_SPECIFIC_ISNEW), 1549 "default_image_recipe" : project.get_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE), 1550 "mru" : Build.objects.all().filter(project=project,outcome=Build.IN_PROGRESS), 1551 } 1552 if project.build_set.filter(outcome=Build.IN_PROGRESS).count() > 0: 1553 context['build_in_progress_none_completed'] = True 1554 else: 1555 context['build_in_progress_none_completed'] = False 1556 return toaster_render(request, "project.html", context) 1557 1558 # perform the final actions for the project specific page 1559 def project_specific_finalize(cmnd, pid): 1560 project = Project.objects.get(pk=pid) 1561 callback = project.get_variable(Project.PROJECT_SPECIFIC_CALLBACK) 1562 if "update" == cmnd: 1563 # Delete all '_PROJECT_PREPARE_' builds 1564 for b in Build.objects.all().filter(project=project): 1565 delete_build = False 1566 for t in b.target_set.all(): 1567 if '_PROJECT_PREPARE_' == t.target: 1568 delete_build = True 1569 if delete_build: 1570 from django.core import management 1571 management.call_command('builddelete', str(b.id), interactive=False) 1572 # perform callback at this last moment if defined, in case Toaster gets shutdown next 1573 default_target = project.get_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE) 1574 if callback: 1575 callback = callback.replace("<IMAGE>",default_target) 1576 if "cancel" == cmnd: 1577 if callback: 1578 callback = callback.replace("<IMAGE>","none") 1579 callback = callback.replace("--update","--cancel") 1580 # perform callback at this last moment if defined, in case this Toaster gets shutdown next 1581 ret = '' 1582 if callback: 1583 ret = os.system('bash -c "%s"' % callback) 1584 project.set_variable(Project.PROJECT_SPECIFIC_CALLBACK,'') 1585 # Delete the temp project specific variables 1586 project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,'') 1587 project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_NONE) 1588 # WORKAROUND: Release this workaround flag 1589 project.set_variable('INTERNAL_PROJECT_SPECIFIC_SKIPRELEASE','') 1590 1591 # Shows the final landing page for project specific update 1592 def landing_specific(request, pid): 1593 project_specific_finalize("update", pid) 1594 context = { 1595 "install_dir": os.environ['TOASTER_DIR'], 1596 } 1597 return toaster_render(request, "landing_specific.html", context) 1598 1599 # Shows the related landing-specific page 1600 def landing_specific_cancel(request, pid): 1601 project_specific_finalize("cancel", pid) 1602 context = { 1603 "install_dir": os.environ['TOASTER_DIR'], 1604 "status": "cancel", 1605 } 1606 return toaster_render(request, "landing_specific.html", context) 1607 1608 def jsunittests(request): 1609 """ Provides a page for the js unit tests """ 1610 bbv = BitbakeVersion.objects.filter(branch="master").first() 1611 release = Release.objects.filter(bitbake_version=bbv).first() 1612 1613 name = "_js_unit_test_prj_" 1614 1615 # If there is an existing project by this name delete it. 1616 # We don't want Lots of duplicates cluttering up the projects. 1617 Project.objects.filter(name=name).delete() 1618 1619 new_project = Project.objects.create_project(name=name, 1620 release=release) 1621 # Add a layer 1622 layer = new_project.get_all_compatible_layer_versions().first() 1623 1624 ProjectLayer.objects.get_or_create(layercommit=layer, 1625 project=new_project) 1626 1627 # make sure we have a machine set for this project 1628 ProjectVariable.objects.get_or_create(project=new_project, 1629 name="MACHINE", 1630 value="qemux86-64") 1631 context = {'project': new_project} 1632 return toaster_render(request, "js-unit-tests.html", context) 1633 1634 from django.views.decorators.csrf import csrf_exempt 1635 @csrf_exempt 1636 @log_view_mixin 1637 def xhr_testreleasechange(request, pid): 1638 def response(data): 1639 return HttpResponse(jsonfilter(data), 1640 content_type="application/json") 1641 1642 """ returns layer versions that would be deleted on the new 1643 release__pk """ 1644 try: 1645 prj = Project.objects.get(pk = pid) 1646 new_release_id = request.GET['new_release_id'] 1647 1648 # If we're already on this project do nothing 1649 if prj.release.pk == int(new_release_id): 1650 return reponse({"error": "ok", "rows": []}) 1651 1652 retval = [] 1653 1654 for project in prj.projectlayer_set.all(): 1655 release = Release.objects.get(pk = new_release_id) 1656 1657 layer_versions = prj.get_all_compatible_layer_versions() 1658 layer_versions = layer_versions.filter(release = release) 1659 layer_versions = layer_versions.filter(layer__name = project.layercommit.layer.name) 1660 1661 # there is no layer_version with the new release id, 1662 # and the same name 1663 if layer_versions.count() < 1: 1664 retval.append(project) 1665 1666 return response({"error":"ok", 1667 "rows": [_lv_to_dict(prj) for y in [x.layercommit for x in retval]] 1668 }) 1669 1670 except Exception as e: 1671 return response({"error": str(e) }) 1672 1673 @log_view_mixin 1674 def xhr_configvaredit(request, pid): 1675 try: 1676 prj = Project.objects.get(id = pid) 1677 # There are cases where user can add variables which hold values 1678 # like http://, file:/// etc. In such case a simple split(":") 1679 # would fail. One example is SSTATE_MIRRORS variable. So we use 1680 # max_split var to handle them. 1681 max_split = 1 1682 # add conf variables 1683 if 'configvarAdd' in request.POST: 1684 t=request.POST['configvarAdd'].strip() 1685 if ":" in t: 1686 variable, value = t.split(":", max_split) 1687 else: 1688 variable = t 1689 value = "" 1690 1691 pt, created = ProjectVariable.objects.get_or_create(project = prj, name = variable, value = value) 1692 # change conf variables 1693 if 'configvarChange' in request.POST: 1694 t=request.POST['configvarChange'].strip() 1695 if ":" in t: 1696 variable, value = t.split(":", max_split) 1697 else: 1698 variable = t 1699 value = "" 1700 1701 pt, created = ProjectVariable.objects.get_or_create(project = prj, name = variable) 1702 pt.value=value 1703 pt.save() 1704 # remove conf variables 1705 if 'configvarDel' in request.POST: 1706 t=request.POST['configvarDel'].strip() 1707 pt = ProjectVariable.objects.get(pk = int(t)).delete() 1708 1709 # return all project settings, filter out disallowed and elsewhere-managed variables 1710 vars_managed,vars_fstypes,vars_disallowed = get_project_configvars_context() 1711 configvars_query = ProjectVariable.objects.filter(project_id = pid).all() 1712 for var in vars_managed: 1713 configvars_query = configvars_query.exclude(name = var) 1714 for var in vars_disallowed: 1715 configvars_query = configvars_query.exclude(name = var) 1716 1717 return_data = { 1718 "error": "ok", 1719 'configvars': [(x.name, x.value, x.pk) for x in configvars_query] 1720 } 1721 try: 1722 return_data['distro'] = ProjectVariable.objects.get(project = prj, name = "DISTRO").value, 1723 except ProjectVariable.DoesNotExist: 1724 pass 1725 try: 1726 return_data['dl_dir'] = ProjectVariable.objects.get(project = prj, name = "DL_DIR").value, 1727 except ProjectVariable.DoesNotExist: 1728 pass 1729 try: 1730 return_data['fstypes'] = ProjectVariable.objects.get(project = prj, name = "IMAGE_FSTYPES").value, 1731 except ProjectVariable.DoesNotExist: 1732 pass 1733 try: 1734 return_data['image_install:append'] = ProjectVariable.objects.get(project = prj, name = "IMAGE_INSTALL:append").value, 1735 except ProjectVariable.DoesNotExist: 1736 pass 1737 try: 1738 return_data['package_classes'] = ProjectVariable.objects.get(project = prj, name = "PACKAGE_CLASSES").value, 1739 except ProjectVariable.DoesNotExist: 1740 pass 1741 try: 1742 return_data['sstate_dir'] = ProjectVariable.objects.get(project = prj, name = "SSTATE_DIR").value, 1743 except ProjectVariable.DoesNotExist: 1744 pass 1745 1746 return HttpResponse(json.dumps( return_data ), content_type = "application/json") 1747 1748 except Exception as e: 1749 return HttpResponse(json.dumps({"error":str(e) + "\n" + traceback.format_exc()}), content_type = "application/json") 1750 1751 1752 @log_view_mixin 1753 def customrecipe_download(request, pid, recipe_id): 1754 recipe = get_object_or_404(CustomImageRecipe, pk=recipe_id) 1755 1756 file_data = recipe.generate_recipe_file_contents() 1757 1758 response = HttpResponse(file_data, content_type='text/plain') 1759 response['Content-Disposition'] = \ 1760 'attachment; filename="%s_%s.bb"' % (recipe.name, 1761 recipe.version) 1762 1763 return response 1764 1765 def importlayer(request, pid): 1766 template = "importlayer.html" 1767 context = { 1768 'project': Project.objects.get(id=pid), 1769 } 1770 return toaster_render(request, template, context) 1771 1772 def layerdetails(request, pid, layerid): 1773 project = Project.objects.get(pk=pid) 1774 layer_version = Layer_Version.objects.get(pk=layerid) 1775 1776 project_layers = ProjectLayer.objects.filter( 1777 project=project).values_list("layercommit_id", 1778 flat=True) 1779 1780 context = { 1781 'project': project, 1782 'layer_source': LayerSource.types_dict(), 1783 'layerversion': layer_version, 1784 'layerdeps': { 1785 "list": [ 1786 { 1787 "id": dep.id, 1788 "name": dep.layer.name, 1789 "layerdetailurl": reverse('layerdetails', 1790 args=(pid, dep.pk)), 1791 "vcs_url": dep.layer.vcs_url, 1792 "vcs_reference": dep.get_vcs_reference() 1793 } 1794 for dep in layer_version.get_alldeps(project.id)] 1795 }, 1796 'projectlayers': list(project_layers) 1797 } 1798 1799 return toaster_render(request, 'layerdetails.html', context) 1800 1801 1802 def get_project_configvars_context(): 1803 # Vars managed outside of this view 1804 vars_managed = { 1805 'MACHINE', 'BBLAYERS' 1806 } 1807 1808 vars_disallowed = { 1809 'PARALLEL_MAKE','BB_NUMBER_THREADS', 1810 'BB_DISKMON_DIRS','BB_NUMBER_THREADS','CVS_PROXY_HOST','CVS_PROXY_PORT', 1811 'PARALLEL_MAKE','TMPDIR', 1812 'all_proxy','ftp_proxy','http_proxy ','https_proxy' 1813 } 1814 1815 vars_fstypes = Target_Image_File.SUFFIXES 1816 1817 return(vars_managed,sorted(vars_fstypes),vars_disallowed) 1818 1819 def projectconf(request, pid): 1820 1821 try: 1822 prj = Project.objects.get(id = pid) 1823 except Project.DoesNotExist: 1824 return HttpResponseNotFound("<h1>Project id " + pid + " is unavailable</h1>") 1825 1826 # remove disallowed and externally managed varaibles from this list 1827 vars_managed,vars_fstypes,vars_disallowed = get_project_configvars_context() 1828 configvars = ProjectVariable.objects.filter(project_id = pid).all() 1829 for var in vars_managed: 1830 configvars = configvars.exclude(name = var) 1831 for var in vars_disallowed: 1832 configvars = configvars.exclude(name = var) 1833 1834 context = { 1835 'project': prj, 1836 'configvars': configvars, 1837 'vars_managed': vars_managed, 1838 'vars_fstypes': vars_fstypes, 1839 'vars_disallowed': vars_disallowed, 1840 } 1841 1842 try: 1843 context['distro'] = ProjectVariable.objects.get(project = prj, name = "DISTRO").value 1844 context['distro_defined'] = "1" 1845 except ProjectVariable.DoesNotExist: 1846 pass 1847 try: 1848 if ProjectVariable.objects.get(project = prj, name = "DL_DIR").value == "${TOPDIR}/../downloads": 1849 be = BuildEnvironment.objects.get(pk = str(1)) 1850 dl_dir = os.path.join(dirname(be.builddir), "downloads") 1851 context['dl_dir'] = dl_dir 1852 pv, created = ProjectVariable.objects.get_or_create(project = prj, name = "DL_DIR") 1853 pv.value = dl_dir 1854 pv.save() 1855 else: 1856 context['dl_dir'] = ProjectVariable.objects.get(project = prj, name = "DL_DIR").value 1857 context['dl_dir_defined'] = "1" 1858 except (ProjectVariable.DoesNotExist, BuildEnvironment.DoesNotExist): 1859 pass 1860 try: 1861 context['fstypes'] = ProjectVariable.objects.get(project = prj, name = "IMAGE_FSTYPES").value 1862 context['fstypes_defined'] = "1" 1863 except ProjectVariable.DoesNotExist: 1864 pass 1865 try: 1866 context['image_install:append'] = ProjectVariable.objects.get(project = prj, name = "IMAGE_INSTALL:append").value 1867 context['image_install_append_defined'] = "1" 1868 except ProjectVariable.DoesNotExist: 1869 pass 1870 try: 1871 context['package_classes'] = ProjectVariable.objects.get(project = prj, name = "PACKAGE_CLASSES").value 1872 context['package_classes_defined'] = "1" 1873 except ProjectVariable.DoesNotExist: 1874 pass 1875 try: 1876 if ProjectVariable.objects.get(project = prj, name = "SSTATE_DIR").value == "${TOPDIR}/../sstate-cache": 1877 be = BuildEnvironment.objects.get(pk = str(1)) 1878 sstate_dir = os.path.join(dirname(be.builddir), "sstate-cache") 1879 context['sstate_dir'] = sstate_dir 1880 pv, created = ProjectVariable.objects.get_or_create(project = prj, name = "SSTATE_DIR") 1881 pv.value = sstate_dir 1882 pv.save() 1883 else: 1884 context['sstate_dir'] = ProjectVariable.objects.get(project = prj, name = "SSTATE_DIR").value 1885 context['sstate_dir_defined'] = "1" 1886 except (ProjectVariable.DoesNotExist, BuildEnvironment.DoesNotExist): 1887 pass 1888 1889 return toaster_render(request, "projectconf.html", context) 1890 1891 def _file_names_for_artifact(build, artifact_type, artifact_id): 1892 """ 1893 Return a tuple (file path, file name for the download response) for an 1894 artifact of type artifact_type with ID artifact_id for build; if 1895 artifact type is not supported, returns (None, None) 1896 """ 1897 file_name = None 1898 response_file_name = None 1899 1900 if artifact_type == "cookerlog": 1901 file_name = build.cooker_log_path 1902 response_file_name = "cooker.log" 1903 1904 elif artifact_type == "imagefile": 1905 file_name = Target_Image_File.objects.get(target__build = build, pk = artifact_id).file_name 1906 1907 elif artifact_type == "targetkernelartifact": 1908 target = TargetKernelFile.objects.get(pk=artifact_id) 1909 file_name = target.file_name 1910 1911 elif artifact_type == "targetsdkartifact": 1912 target = TargetSDKFile.objects.get(pk=artifact_id) 1913 file_name = target.file_name 1914 1915 elif artifact_type == "licensemanifest": 1916 file_name = Target.objects.get(build = build, pk = artifact_id).license_manifest_path 1917 1918 elif artifact_type == "packagemanifest": 1919 file_name = Target.objects.get(build = build, pk = artifact_id).package_manifest_path 1920 1921 elif artifact_type == "tasklogfile": 1922 file_name = Task.objects.get(build = build, pk = artifact_id).logfile 1923 1924 elif artifact_type == "logmessagefile": 1925 file_name = LogMessage.objects.get(build = build, pk = artifact_id).pathname 1926 1927 if file_name and not response_file_name: 1928 response_file_name = os.path.basename(file_name) 1929 1930 return (file_name, response_file_name) 1931 1932 def build_artifact(request, build_id, artifact_type, artifact_id): 1933 """ 1934 View which returns a build artifact file as a response 1935 """ 1936 file_name = None 1937 response_file_name = None 1938 1939 try: 1940 build = Build.objects.get(pk = build_id) 1941 file_name, response_file_name = _file_names_for_artifact( 1942 build, artifact_type, artifact_id 1943 ) 1944 1945 if file_name and response_file_name: 1946 fsock = open(file_name, "rb") 1947 content_type = MimeTypeFinder.get_mimetype(file_name) 1948 1949 response = HttpResponse(fsock, content_type = content_type) 1950 1951 disposition = "attachment; filename=" + response_file_name 1952 response["Content-Disposition"] = disposition 1953 1954 return response 1955 else: 1956 return toaster_render(request, "unavailable_artifact.html") 1957 except (ObjectDoesNotExist, IOError): 1958 return toaster_render(request, "unavailable_artifact.html") 1959 1960 1961class CommandLineBuilds(TemplateView): 1962 model = EventLogsImports 1963 template_name = 'command_line_builds.html' 1964 1965 def get_context_data(self, **kwargs): 1966 context = super(CommandLineBuilds, self).get_context_data(**kwargs) 1967 #get value from BB_DEFAULT_EVENTLOG defined in bitbake.conf 1968 eventlog = subprocess.check_output(['bitbake-getvar', 'BB_DEFAULT_EVENTLOG', '--value']) 1969 if eventlog: 1970 logs_dir = os.path.dirname(eventlog.decode().strip('\n')) 1971 files = os.listdir(logs_dir) 1972 imported_files = EventLogsImports.objects.all() 1973 files_list = [] 1974 1975 # Filter files that end with ".json" 1976 event_files = [] 1977 for file in files: 1978 if file.endswith(".json"): 1979 # because BB_DEFAULT_EVENTLOG is a directory, we need to check if the file is a valid eventlog 1980 with open("{}/{}".format(logs_dir, file)) as efile: 1981 content = efile.read() 1982 if 'allvariables' in content: 1983 event_files.append(file) 1984 1985 #build dict for template using db data 1986 for event_file in event_files: 1987 if imported_files.filter(name=event_file): 1988 files_list.append({ 1989 'name': event_file, 1990 'imported': True, 1991 'build_id': imported_files.filter(name=event_file)[0].build_id, 1992 'size': os.path.getsize("{}/{}".format(logs_dir, event_file)) 1993 }) 1994 else: 1995 files_list.append({ 1996 'name': event_file, 1997 'imported': False, 1998 'build_id': None, 1999 'size': os.path.getsize("{}/{}".format(logs_dir, event_file)) 2000 }) 2001 context['import_all'] = True 2002 2003 context['files'] = files_list 2004 context['dir'] = logs_dir 2005 else: 2006 context['files'] = [] 2007 context['dir'] = '' 2008 2009 # enable session variable 2010 if not self.request.session.get('file'): 2011 self.request.session['file'] = "" 2012 2013 context['form'] = LoadFileForm() 2014 context['project_enable'] = project_enable 2015 return context 2016 2017 def post(self, request, **kwargs): 2018 logs_dir = request.POST.get('dir') 2019 all_files = request.POST.get('all') 2020 2021 # check if a build is already in progress 2022 if Build.objects.filter(outcome=Build.IN_PROGRESS): 2023 messages.add_message( 2024 self.request, 2025 messages.ERROR, 2026 "A build is already in progress. Please wait for it to complete before starting a new build." 2027 ) 2028 return JsonResponse({'response': 'building'}) 2029 imported_files = EventLogsImports.objects.all() 2030 try: 2031 if all_files == 'true': 2032 # use of session variable to deactivate icon for builds in progress 2033 request.session['all_builds'] = True 2034 request.session.modified = True 2035 request.session.save() 2036 2037 files = ast.literal_eval(request.POST.get('file')) 2038 for file in files: 2039 if imported_files.filter(name=file.get('name')).exists(): 2040 imported_files.filter(name=file.get('name'))[0].imported = True 2041 else: 2042 with open("{}/{}".format(logs_dir, file.get('name'))) as eventfile: 2043 # load variables from the first line 2044 variables = None 2045 while line := eventfile.readline().strip(): 2046 try: 2047 variables = json.loads(line)['allvariables'] 2048 break 2049 except (KeyError, json.JSONDecodeError): 2050 continue 2051 if not variables: 2052 raise Exception("File content missing build variables") 2053 eventfile.seek(0) 2054 params = namedtuple('ConfigParams', ['observe_only'])(True) 2055 player = eventreplay.EventPlayer(eventfile, variables) 2056 2057 toasterui.main(player, player, params) 2058 event_log_import = EventLogsImports.objects.create(name=file.get('name'), imported=True) 2059 event_log_import.build_id = Build.objects.last().id 2060 event_log_import.save() 2061 else: 2062 if self.request.FILES.get('eventlog_file'): 2063 file = self.request.FILES['eventlog_file'] 2064 else: 2065 file = request.POST.get('file') 2066 # use of session variable to deactivate icon for build in progress 2067 request.session['file'] = file 2068 request.session['all_builds'] = False 2069 request.session.modified = True 2070 request.session.save() 2071 2072 if imported_files.filter(name=file).exists(): 2073 imported_files.filter(name=file)[0].imported = True 2074 else: 2075 if isinstance(file, InMemoryUploadedFile) or isinstance(file, TemporaryUploadedFile): 2076 variables = None 2077 while line := file.readline().strip(): 2078 try: 2079 variables = json.loads(line)['allvariables'] 2080 break 2081 except (KeyError, json.JSONDecodeError): 2082 continue 2083 if not variables: 2084 raise Exception("File content missing build variables") 2085 file.seek(0) 2086 params = namedtuple('ConfigParams', ['observe_only'])(True) 2087 player = eventreplay.EventPlayer(file, variables) 2088 if not os.path.exists('{}/{}'.format(logs_dir, file.name)): 2089 fs = FileSystemStorage(location=logs_dir) 2090 fs.save(file.name, file) 2091 toasterui.main(player, player, params) 2092 else: 2093 with open("{}/{}".format(logs_dir, file)) as eventfile: 2094 # load variables from the first line 2095 variables = None 2096 while line := eventfile.readline().strip(): 2097 try: 2098 variables = json.loads(line)['allvariables'] 2099 break 2100 except (KeyError, json.JSONDecodeError): 2101 continue 2102 if not variables: 2103 raise Exception("File content missing build variables") 2104 eventfile.seek(0) 2105 params = namedtuple('ConfigParams', ['observe_only'])(True) 2106 player = eventreplay.EventPlayer(eventfile, variables) 2107 toasterui.main(player, player, params) 2108 event_log_import = EventLogsImports.objects.create(name=file, imported=True) 2109 event_log_import.build_id = Build.objects.last().id 2110 event_log_import.save() 2111 request.session['file'] = "" 2112 except Exception: 2113 messages.add_message( 2114 self.request, 2115 messages.ERROR, 2116 "The file content is not in the correct format. Update file content or upload a different file." 2117 ) 2118 return HttpResponseRedirect("/toastergui/cmdline/") 2119 return HttpResponseRedirect('/toastergui/builds/') 2120