1# 2# BitBake Toaster Implementation 3# 4# Copyright (C) 2015 Intel Corporation 5# 6# SPDX-License-Identifier: GPL-2.0-only 7# 8 9from django.views.generic import View, TemplateView 10from django.views.decorators.cache import cache_control 11from django.shortcuts import HttpResponse 12from django.core.cache import cache 13from django.core.paginator import Paginator, EmptyPage 14from django.db.models import Q 15from orm.models import Project, Build 16from django.template import Context, Template 17from django.template import VariableDoesNotExist 18from django.template import TemplateSyntaxError 19from django.core.serializers.json import DjangoJSONEncoder 20from django.core.exceptions import FieldError 21from django.utils import timezone 22from toastergui.templatetags.projecttags import sectohms, get_tasks 23from toastergui.templatetags.projecttags import json as template_json 24from django.http import JsonResponse 25from django.urls import reverse 26 27import types 28import json 29import collections 30import re 31import os 32 33from toastergui.tablefilter import TableFilterMap 34 35try: 36 from urllib import unquote_plus 37except ImportError: 38 from urllib.parse import unquote_plus 39 40import logging 41logger = logging.getLogger("toaster") 42 43 44class NoFieldOrDataName(Exception): 45 pass 46 47 48class ToasterTable(TemplateView): 49 def __init__(self, *args, **kwargs): 50 super(ToasterTable, self).__init__() 51 if 'template_name' in kwargs: 52 self.template_name = kwargs['template_name'] 53 self.title = "Table" 54 self.queryset = None 55 self.columns = [] 56 57 # map from field names to Filter instances 58 self.filter_map = TableFilterMap() 59 60 self.total_count = 0 61 self.static_context_extra = {} 62 self.empty_state = "Sorry - no data found" 63 self.default_orderby = "" 64 65 # prevent HTTP caching of table data 66 @cache_control(must_revalidate=True, 67 max_age=0, no_store=True, no_cache=True) 68 def dispatch(self, *args, **kwargs): 69 return super(ToasterTable, self).dispatch(*args, **kwargs) 70 71 def get_context_data(self, **kwargs): 72 context = super(ToasterTable, self).get_context_data(**kwargs) 73 context['title'] = self.title 74 context['table_name'] = type(self).__name__.lower() 75 context['empty_state'] = self.empty_state 76 77 # global variables 78 context['project_enable'] = ('1' == os.environ.get('TOASTER_BUILDSERVER')) 79 try: 80 context['project_specific'] = ('1' == os.environ.get('TOASTER_PROJECTSPECIFIC')) 81 except: 82 context['project_specific'] = '' 83 84 return context 85 86 def get(self, request, *args, **kwargs): 87 if request.GET.get('format', None) == 'json': 88 89 self.setup_queryset(*args, **kwargs) 90 # Put the project id into the context for the static_data_template 91 if 'pid' in kwargs: 92 self.static_context_extra['pid'] = kwargs['pid'] 93 94 cmd = request.GET.get('cmd', None) 95 if cmd and 'filterinfo' in cmd: 96 data = self.get_filter_info(request, **kwargs) 97 else: 98 # If no cmd is specified we give you the table data 99 data = self.get_data(request, **kwargs) 100 101 return HttpResponse(data, content_type="application/json") 102 103 return super(ToasterTable, self).get(request, *args, **kwargs) 104 105 def get_filter_info(self, request, **kwargs): 106 self.setup_filters(**kwargs) 107 108 search = request.GET.get("search", None) 109 if search: 110 self.apply_search(search) 111 112 name = request.GET.get("name", None) 113 table_filter = self.filter_map.get_filter(name) 114 return json.dumps(table_filter.to_json(self.queryset), 115 indent=2, 116 cls=DjangoJSONEncoder) 117 118 def setup_columns(self, *args, **kwargs): 119 """ function to implement in the subclass which sets up 120 the columns """ 121 pass 122 123 def setup_filters(self, *args, **kwargs): 124 """ function to implement in the subclass which sets up the 125 filters """ 126 pass 127 128 def setup_queryset(self, *args, **kwargs): 129 """ function to implement in the subclass which sets up the 130 queryset""" 131 pass 132 133 def add_filter(self, table_filter): 134 """Add a filter to the table. 135 136 Args: 137 table_filter: Filter instance 138 """ 139 self.filter_map.add_filter(table_filter.name, table_filter) 140 141 def add_column(self, title="", help_text="", 142 orderable=False, hideable=True, hidden=False, 143 field_name="", filter_name=None, static_data_name=None, 144 static_data_template=None): 145 """Add a column to the table. 146 147 Args: 148 title (str): Title for the table header 149 help_text (str): Optional help text to describe the column 150 orderable (bool): Whether the column can be ordered. 151 We order on the field_name. 152 hideable (bool): Whether the user can hide the column 153 hidden (bool): Whether the column is default hidden 154 field_name (str or list): field(s) required for this column's data 155 static_data_name (str, optional): The column's main identifier 156 which will replace the field_name. 157 static_data_template(str, optional): The template to be rendered 158 as data 159 """ 160 161 self.columns.append({'title': title, 162 'help_text': help_text, 163 'orderable': orderable, 164 'hideable': hideable, 165 'hidden': hidden, 166 'field_name': field_name, 167 'filter_name': filter_name, 168 'static_data_name': static_data_name, 169 'static_data_template': static_data_template}) 170 171 def set_column_hidden(self, title, hidden): 172 """ 173 Set the hidden state of the column to the value of hidden 174 """ 175 for col in self.columns: 176 if col['title'] == title: 177 col['hidden'] = hidden 178 break 179 180 def set_column_hideable(self, title, hideable): 181 """ 182 Set the hideable state of the column to the value of hideable 183 """ 184 for col in self.columns: 185 if col['title'] == title: 186 col['hideable'] = hideable 187 break 188 189 def render_static_data(self, template, row): 190 """Utility function to render the static data template""" 191 192 context = { 193 'extra': self.static_context_extra, 194 'data': row, 195 } 196 197 context = Context(context) 198 template = Template(template) 199 200 return template.render(context) 201 202 def apply_filter(self, filters, filter_value, **kwargs): 203 """ 204 Apply a filter submitted in the querystring to the ToasterTable 205 206 filters: (str) in the format: 207 '<filter name>:<action name>' 208 filter_value: (str) parameters to pass to the named filter 209 210 <filter name> and <action name> are used to look up the correct filter 211 in the ToasterTable's filter map; the <action params> are set on 212 TableFilterAction* before its filter is applied and may modify the 213 queryset returned by the filter 214 """ 215 self.setup_filters(**kwargs) 216 217 try: 218 filter_name, action_name = filters.split(':') 219 action_params = unquote_plus(filter_value) 220 except ValueError: 221 return 222 223 if "all" in action_name: 224 return 225 226 try: 227 table_filter = self.filter_map.get_filter(filter_name) 228 action = table_filter.get_action(action_name) 229 action.set_filter_params(action_params) 230 self.queryset = action.filter(self.queryset) 231 except KeyError: 232 # pass it to the user - programming error here 233 raise 234 235 def apply_orderby(self, orderby): 236 # Note that django will execute this when we try to retrieve the data 237 self.queryset = self.queryset.order_by(orderby) 238 239 def apply_search(self, search_term): 240 """Creates a query based on the model's search_allowed_fields""" 241 242 if not hasattr(self.queryset.model, 'search_allowed_fields'): 243 raise Exception("Search fields aren't defined in the model %s" 244 % self.queryset.model) 245 246 search_queries = None 247 for st in search_term.split(" "): 248 queries = None 249 for field in self.queryset.model.search_allowed_fields: 250 query = Q(**{field + '__icontains': st}) 251 if queries: 252 queries |= query 253 else: 254 queries = query 255 256 if search_queries: 257 search_queries &= queries 258 else: 259 search_queries = queries 260 261 self.queryset = self.queryset.filter(search_queries) 262 263 def get_data(self, request, **kwargs): 264 """ 265 Returns the data for the page requested with the specified 266 parameters applied 267 268 filters: filter and action name, e.g. "outcome:build_succeeded" 269 filter_value: value to pass to the named filter+action, e.g. "on" 270 (for a toggle filter) or "2015-12-11,2015-12-12" 271 (for a date range filter) 272 """ 273 274 page_num = request.GET.get("page", 1) 275 limit = request.GET.get("limit", 10) 276 search = request.GET.get("search", None) 277 filters = request.GET.get("filter", None) 278 filter_value = request.GET.get("filter_value", "on") 279 orderby = request.GET.get("orderby", None) 280 nocache = request.GET.get("nocache", None) 281 282 # Make a unique cache name 283 cache_name = self.__class__.__name__ 284 285 for key, val in request.GET.items(): 286 if key == 'nocache': 287 continue 288 cache_name = cache_name + str(key) + str(val) 289 290 for key, val in kwargs.items(): 291 cache_name = cache_name + str(key) + str(val) 292 293 # No special chars allowed in the cache name apart from dash 294 cache_name = re.sub(r'[^A-Za-z0-9-]', "", cache_name) 295 296 if nocache: 297 cache.delete(cache_name) 298 299 data = cache.get(cache_name) 300 301 if data: 302 logger.debug("Got cache data for table '%s'" % self.title) 303 return data 304 305 self.setup_columns(**kwargs) 306 307 if search: 308 self.apply_search(search) 309 if filters: 310 self.apply_filter(filters, filter_value, **kwargs) 311 if orderby: 312 self.apply_orderby(orderby) 313 314 paginator = Paginator(self.queryset, limit) 315 316 try: 317 page = paginator.page(page_num) 318 except EmptyPage: 319 page = paginator.page(1) 320 321 data = { 322 'total': self.queryset.count(), 323 'default_orderby': self.default_orderby, 324 'columns': self.columns, 325 'rows': [], 326 'error': "ok", 327 } 328 329 try: 330 for model_obj in page.object_list: 331 # Use collection to maintain the order 332 required_data = collections.OrderedDict() 333 334 for col in self.columns: 335 field = col['field_name'] 336 if not field: 337 field = col['static_data_name'] 338 if not field: 339 raise NoFieldOrDataName("Must supply a field_name or" 340 "static_data_name for column" 341 "%s.%s" % 342 (self.__class__.__name__, col) 343 ) 344 345 # Check if we need to process some static data 346 if "static_data_name" in col and col['static_data_name']: 347 # Overwrite the field_name with static_data_name 348 # so that this can be used as the html class name 349 col['field_name'] = col['static_data_name'] 350 351 try: 352 # Render the template given 353 required_data[col['static_data_name']] = \ 354 self.render_static_data( 355 col['static_data_template'], model_obj) 356 except (TemplateSyntaxError, 357 VariableDoesNotExist) as e: 358 logger.error("could not render template code" 359 "%s %s %s", 360 col['static_data_template'], 361 e, self.__class__.__name__) 362 required_data[col['static_data_name']] =\ 363 '<!--error-->' 364 365 else: 366 # Traverse to any foriegn key in the field 367 # e.g. recipe__layer_version__name 368 model_data = None 369 370 if "__" in field: 371 for subfield in field.split("__"): 372 if not model_data: 373 # The first iteration is always going to 374 # be on the actual model object instance. 375 # Subsequent ones are on the result of 376 # that. e.g. forieng key objects 377 model_data = getattr(model_obj, 378 subfield) 379 else: 380 model_data = getattr(model_data, 381 subfield) 382 383 else: 384 model_data = getattr(model_obj, 385 col['field_name']) 386 387 # We might have a model function as the field so 388 # call it to return the data needed 389 if isinstance(model_data, types.MethodType): 390 model_data = model_data() 391 392 required_data[col['field_name']] = model_data 393 394 data['rows'].append(required_data) 395 396 except FieldError: 397 # pass it to the user - programming-error here 398 raise 399 400 data = json.dumps(data, indent=2, cls=DjangoJSONEncoder) 401 cache.set(cache_name, data, 60*30) 402 403 return data 404 405 406class ToasterTypeAhead(View): 407 """ A typeahead mechanism to support the front end typeahead widgets """ 408 MAX_RESULTS = 6 409 410 class MissingFieldsException(Exception): 411 pass 412 413 def __init__(self, *args, **kwargs): 414 super(ToasterTypeAhead, self).__init__() 415 416 def get(self, request, *args, **kwargs): 417 def response(data): 418 return HttpResponse(json.dumps(data, 419 indent=2, 420 cls=DjangoJSONEncoder), 421 content_type="application/json") 422 423 error = "ok" 424 425 search_term = request.GET.get("search", None) 426 if search_term is None: 427 # We got no search value so return empty reponse 428 return response({'error': error, 'results': []}) 429 430 try: 431 prj = Project.objects.get(pk=kwargs['pid']) 432 except KeyError: 433 prj = None 434 435 results = self.apply_search(search_term, 436 prj, 437 request)[:ToasterTypeAhead.MAX_RESULTS] 438 439 if len(results) > 0: 440 try: 441 self.validate_fields(results[0]) 442 except self.MissingFieldsException as e: 443 error = e 444 445 data = {'results': results, 446 'error': error} 447 448 return response(data) 449 450 def validate_fields(self, result): 451 if 'name' in result is False or 'detail' in result is False: 452 raise self.MissingFieldsException( 453 "name and detail are required fields") 454 455 def apply_search(self, search_term, prj): 456 """ Override this function to implement search. Return an array of 457 dictionaries with a minium of a name and detail field""" 458 pass 459 460 461class MostRecentBuildsView(View): 462 def _was_yesterday_or_earlier(self, completed_on): 463 now = timezone.now() 464 delta = now - completed_on 465 466 if delta.days >= 1: 467 return True 468 469 return False 470 471 def get(self, request, *args, **kwargs): 472 """ 473 Returns a list of builds in JSON format. 474 """ 475 project = None 476 477 project_id = request.GET.get('project_id', None) 478 if project_id: 479 try: 480 project = Project.objects.get(pk=project_id) 481 except: 482 # if project lookup fails, assume no project 483 pass 484 485 recent_build_objs = Build.get_recent(project) 486 recent_builds = [] 487 488 for build_obj in recent_build_objs: 489 dashboard_url = reverse('builddashboard', args=(build_obj.pk,)) 490 buildtime_url = reverse('buildtime', args=(build_obj.pk,)) 491 rebuild_url = \ 492 reverse('xhr_buildrequest', args=(build_obj.project.pk,)) 493 cancel_url = \ 494 reverse('xhr_buildrequest', args=(build_obj.project.pk,)) 495 496 build = {} 497 build['id'] = build_obj.pk 498 build['dashboard_url'] = dashboard_url 499 500 buildrequest_id = None 501 if hasattr(build_obj, 'buildrequest'): 502 buildrequest_id = build_obj.buildrequest.pk 503 build['buildrequest_id'] = buildrequest_id 504 505 if build_obj.recipes_to_parse > 0: 506 build['recipes_parsed_percentage'] = \ 507 int((build_obj.recipes_parsed / 508 build_obj.recipes_to_parse) * 100) 509 else: 510 build['recipes_parsed_percentage'] = 0 511 if build_obj.repos_to_clone > 0: 512 build['repos_cloned_percentage'] = \ 513 int((build_obj.repos_cloned / 514 build_obj.repos_to_clone) * 100) 515 else: 516 build['repos_cloned_percentage'] = 0 517 518 build['progress_item'] = build_obj.progress_item 519 520 tasks_complete_percentage = 0 521 if build_obj.outcome in (Build.SUCCEEDED, Build.FAILED): 522 tasks_complete_percentage = 100 523 elif build_obj.outcome == Build.IN_PROGRESS: 524 tasks_complete_percentage = build_obj.completeper() 525 build['tasks_complete_percentage'] = tasks_complete_percentage 526 527 build['state'] = build_obj.get_state() 528 529 build['errors'] = build_obj.errors.count() 530 build['dashboard_errors_url'] = dashboard_url + '#errors' 531 532 build['warnings'] = build_obj.warnings.count() 533 build['dashboard_warnings_url'] = dashboard_url + '#warnings' 534 535 build['buildtime'] = sectohms(build_obj.timespent_seconds) 536 build['buildtime_url'] = buildtime_url 537 538 build['rebuild_url'] = rebuild_url 539 build['cancel_url'] = cancel_url 540 541 build['is_default_project_build'] = build_obj.project.is_default 542 543 build['build_targets_json'] = \ 544 template_json(get_tasks(build_obj.target_set.all())) 545 546 # convert completed_on time to user's timezone 547 completed_on = timezone.localtime(build_obj.completed_on) 548 549 completed_on_template = '%H:%M' 550 if self._was_yesterday_or_earlier(completed_on): 551 completed_on_template = '%d/%m/%Y ' + completed_on_template 552 build['completed_on'] = completed_on.strftime( 553 completed_on_template) 554 555 targets = [] 556 target_objs = build_obj.get_sorted_target_list() 557 for target_obj in target_objs: 558 if target_obj.task: 559 targets.append(target_obj.target + ':' + target_obj.task) 560 else: 561 targets.append(target_obj.target) 562 build['targets'] = ' '.join(targets) 563 564 # abbreviated form of the full target list 565 abbreviated_targets = '' 566 num_targets = len(targets) 567 if num_targets > 0: 568 abbreviated_targets = targets[0] 569 if num_targets > 1: 570 abbreviated_targets += (' +%s' % (num_targets - 1)) 571 build['targets_abbreviated'] = abbreviated_targets 572 573 recent_builds.append(build) 574 575 return JsonResponse(recent_builds, safe=False) 576