1# 2# ex:ts=4:sw=4:sts=4:et 3# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- 4# 5# BitBake Toaster Implementation 6# 7# Copyright (C) 2015 Intel Corporation 8# 9# This program is free software; you can redistribute it and/or modify 10# it under the terms of the GNU General Public License version 2 as 11# published by the Free Software Foundation. 12# 13# This program is distributed in the hope that it will be useful, 14# but WITHOUT ANY WARRANTY; without even the implied warranty of 15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16# GNU General Public License for more details. 17# 18# You should have received a copy of the GNU General Public License along 19# with this program; if not, write to the Free Software Foundation, Inc., 20# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 21 22from django.views.generic import View, TemplateView 23from django.views.decorators.cache import cache_control 24from django.shortcuts import HttpResponse 25from django.core.cache import cache 26from django.core.paginator import Paginator, EmptyPage 27from django.db.models import Q 28from orm.models import Project, Build 29from django.template import Context, Template 30from django.template import VariableDoesNotExist 31from django.template import TemplateSyntaxError 32from django.core.serializers.json import DjangoJSONEncoder 33from django.core.exceptions import FieldError 34from django.utils import timezone 35from toastergui.templatetags.projecttags import sectohms, get_tasks 36from toastergui.templatetags.projecttags import json as template_json 37from django.http import JsonResponse 38from django.core.urlresolvers import reverse 39 40import types 41import json 42import collections 43import re 44import os 45 46from toastergui.tablefilter import TableFilterMap 47 48try: 49 from urllib import unquote_plus 50except ImportError: 51 from urllib.parse import unquote_plus 52 53import logging 54logger = logging.getLogger("toaster") 55 56 57class NoFieldOrDataName(Exception): 58 pass 59 60 61class ToasterTable(TemplateView): 62 def __init__(self, *args, **kwargs): 63 super(ToasterTable, self).__init__() 64 if 'template_name' in kwargs: 65 self.template_name = kwargs['template_name'] 66 self.title = "Table" 67 self.queryset = None 68 self.columns = [] 69 70 # map from field names to Filter instances 71 self.filter_map = TableFilterMap() 72 73 self.total_count = 0 74 self.static_context_extra = {} 75 self.empty_state = "Sorry - no data found" 76 self.default_orderby = "" 77 78 # prevent HTTP caching of table data 79 @cache_control(must_revalidate=True, 80 max_age=0, no_store=True, no_cache=True) 81 def dispatch(self, *args, **kwargs): 82 return super(ToasterTable, self).dispatch(*args, **kwargs) 83 84 def get_context_data(self, **kwargs): 85 context = super(ToasterTable, self).get_context_data(**kwargs) 86 context['title'] = self.title 87 context['table_name'] = type(self).__name__.lower() 88 context['empty_state'] = self.empty_state 89 90 # global variables 91 context['project_enable'] = ('1' == os.environ.get('TOASTER_BUILDSERVER')) 92 try: 93 context['project_specific'] = ('1' == os.environ.get('TOASTER_PROJECTSPECIFIC')) 94 except: 95 context['project_specific'] = '' 96 97 return context 98 99 def get(self, request, *args, **kwargs): 100 if request.GET.get('format', None) == 'json': 101 102 self.setup_queryset(*args, **kwargs) 103 # Put the project id into the context for the static_data_template 104 if 'pid' in kwargs: 105 self.static_context_extra['pid'] = kwargs['pid'] 106 107 cmd = request.GET.get('cmd', None) 108 if cmd and 'filterinfo' in cmd: 109 data = self.get_filter_info(request, **kwargs) 110 else: 111 # If no cmd is specified we give you the table data 112 data = self.get_data(request, **kwargs) 113 114 return HttpResponse(data, content_type="application/json") 115 116 return super(ToasterTable, self).get(request, *args, **kwargs) 117 118 def get_filter_info(self, request, **kwargs): 119 self.setup_filters(**kwargs) 120 121 search = request.GET.get("search", None) 122 if search: 123 self.apply_search(search) 124 125 name = request.GET.get("name", None) 126 table_filter = self.filter_map.get_filter(name) 127 return json.dumps(table_filter.to_json(self.queryset), 128 indent=2, 129 cls=DjangoJSONEncoder) 130 131 def setup_columns(self, *args, **kwargs): 132 """ function to implement in the subclass which sets up 133 the columns """ 134 pass 135 136 def setup_filters(self, *args, **kwargs): 137 """ function to implement in the subclass which sets up the 138 filters """ 139 pass 140 141 def setup_queryset(self, *args, **kwargs): 142 """ function to implement in the subclass which sets up the 143 queryset""" 144 pass 145 146 def add_filter(self, table_filter): 147 """Add a filter to the table. 148 149 Args: 150 table_filter: Filter instance 151 """ 152 self.filter_map.add_filter(table_filter.name, table_filter) 153 154 def add_column(self, title="", help_text="", 155 orderable=False, hideable=True, hidden=False, 156 field_name="", filter_name=None, static_data_name=None, 157 static_data_template=None): 158 """Add a column to the table. 159 160 Args: 161 title (str): Title for the table header 162 help_text (str): Optional help text to describe the column 163 orderable (bool): Whether the column can be ordered. 164 We order on the field_name. 165 hideable (bool): Whether the user can hide the column 166 hidden (bool): Whether the column is default hidden 167 field_name (str or list): field(s) required for this column's data 168 static_data_name (str, optional): The column's main identifier 169 which will replace the field_name. 170 static_data_template(str, optional): The template to be rendered 171 as data 172 """ 173 174 self.columns.append({'title': title, 175 'help_text': help_text, 176 'orderable': orderable, 177 'hideable': hideable, 178 'hidden': hidden, 179 'field_name': field_name, 180 'filter_name': filter_name, 181 'static_data_name': static_data_name, 182 'static_data_template': static_data_template}) 183 184 def set_column_hidden(self, title, hidden): 185 """ 186 Set the hidden state of the column to the value of hidden 187 """ 188 for col in self.columns: 189 if col['title'] == title: 190 col['hidden'] = hidden 191 break 192 193 def set_column_hideable(self, title, hideable): 194 """ 195 Set the hideable state of the column to the value of hideable 196 """ 197 for col in self.columns: 198 if col['title'] == title: 199 col['hideable'] = hideable 200 break 201 202 def render_static_data(self, template, row): 203 """Utility function to render the static data template""" 204 205 context = { 206 'extra': self.static_context_extra, 207 'data': row, 208 } 209 210 context = Context(context) 211 template = Template(template) 212 213 return template.render(context) 214 215 def apply_filter(self, filters, filter_value, **kwargs): 216 """ 217 Apply a filter submitted in the querystring to the ToasterTable 218 219 filters: (str) in the format: 220 '<filter name>:<action name>' 221 filter_value: (str) parameters to pass to the named filter 222 223 <filter name> and <action name> are used to look up the correct filter 224 in the ToasterTable's filter map; the <action params> are set on 225 TableFilterAction* before its filter is applied and may modify the 226 queryset returned by the filter 227 """ 228 self.setup_filters(**kwargs) 229 230 try: 231 filter_name, action_name = filters.split(':') 232 action_params = unquote_plus(filter_value) 233 except ValueError: 234 return 235 236 if "all" in action_name: 237 return 238 239 try: 240 table_filter = self.filter_map.get_filter(filter_name) 241 action = table_filter.get_action(action_name) 242 action.set_filter_params(action_params) 243 self.queryset = action.filter(self.queryset) 244 except KeyError: 245 # pass it to the user - programming error here 246 raise 247 248 def apply_orderby(self, orderby): 249 # Note that django will execute this when we try to retrieve the data 250 self.queryset = self.queryset.order_by(orderby) 251 252 def apply_search(self, search_term): 253 """Creates a query based on the model's search_allowed_fields""" 254 255 if not hasattr(self.queryset.model, 'search_allowed_fields'): 256 raise Exception("Search fields aren't defined in the model %s" 257 % self.queryset.model) 258 259 search_queries = None 260 for st in search_term.split(" "): 261 queries = None 262 for field in self.queryset.model.search_allowed_fields: 263 query = Q(**{field + '__icontains': st}) 264 if queries: 265 queries |= query 266 else: 267 queries = query 268 269 if search_queries: 270 search_queries &= queries 271 else: 272 search_queries = queries 273 274 self.queryset = self.queryset.filter(search_queries) 275 276 def get_data(self, request, **kwargs): 277 """ 278 Returns the data for the page requested with the specified 279 parameters applied 280 281 filters: filter and action name, e.g. "outcome:build_succeeded" 282 filter_value: value to pass to the named filter+action, e.g. "on" 283 (for a toggle filter) or "2015-12-11,2015-12-12" 284 (for a date range filter) 285 """ 286 287 page_num = request.GET.get("page", 1) 288 limit = request.GET.get("limit", 10) 289 search = request.GET.get("search", None) 290 filters = request.GET.get("filter", None) 291 filter_value = request.GET.get("filter_value", "on") 292 orderby = request.GET.get("orderby", None) 293 nocache = request.GET.get("nocache", None) 294 295 # Make a unique cache name 296 cache_name = self.__class__.__name__ 297 298 for key, val in request.GET.items(): 299 if key == 'nocache': 300 continue 301 cache_name = cache_name + str(key) + str(val) 302 303 for key, val in kwargs.items(): 304 cache_name = cache_name + str(key) + str(val) 305 306 # No special chars allowed in the cache name apart from dash 307 cache_name = re.sub(r'[^A-Za-z0-9-]', "", cache_name) 308 309 if nocache: 310 cache.delete(cache_name) 311 312 data = cache.get(cache_name) 313 314 if data: 315 logger.debug("Got cache data for table '%s'" % self.title) 316 return data 317 318 self.setup_columns(**kwargs) 319 320 if search: 321 self.apply_search(search) 322 if filters: 323 self.apply_filter(filters, filter_value, **kwargs) 324 if orderby: 325 self.apply_orderby(orderby) 326 327 paginator = Paginator(self.queryset, limit) 328 329 try: 330 page = paginator.page(page_num) 331 except EmptyPage: 332 page = paginator.page(1) 333 334 data = { 335 'total': self.queryset.count(), 336 'default_orderby': self.default_orderby, 337 'columns': self.columns, 338 'rows': [], 339 'error': "ok", 340 } 341 342 try: 343 for model_obj in page.object_list: 344 # Use collection to maintain the order 345 required_data = collections.OrderedDict() 346 347 for col in self.columns: 348 field = col['field_name'] 349 if not field: 350 field = col['static_data_name'] 351 if not field: 352 raise NoFieldOrDataName("Must supply a field_name or" 353 "static_data_name for column" 354 "%s.%s" % 355 (self.__class__.__name__, col) 356 ) 357 358 # Check if we need to process some static data 359 if "static_data_name" in col and col['static_data_name']: 360 # Overwrite the field_name with static_data_name 361 # so that this can be used as the html class name 362 col['field_name'] = col['static_data_name'] 363 364 try: 365 # Render the template given 366 required_data[col['static_data_name']] = \ 367 self.render_static_data( 368 col['static_data_template'], model_obj) 369 except (TemplateSyntaxError, 370 VariableDoesNotExist) as e: 371 logger.error("could not render template code" 372 "%s %s %s", 373 col['static_data_template'], 374 e, self.__class__.__name__) 375 required_data[col['static_data_name']] =\ 376 '<!--error-->' 377 378 else: 379 # Traverse to any foriegn key in the field 380 # e.g. recipe__layer_version__name 381 model_data = None 382 383 if "__" in field: 384 for subfield in field.split("__"): 385 if not model_data: 386 # The first iteration is always going to 387 # be on the actual model object instance. 388 # Subsequent ones are on the result of 389 # that. e.g. forieng key objects 390 model_data = getattr(model_obj, 391 subfield) 392 else: 393 model_data = getattr(model_data, 394 subfield) 395 396 else: 397 model_data = getattr(model_obj, 398 col['field_name']) 399 400 # We might have a model function as the field so 401 # call it to return the data needed 402 if isinstance(model_data, types.MethodType): 403 model_data = model_data() 404 405 required_data[col['field_name']] = model_data 406 407 data['rows'].append(required_data) 408 409 except FieldError: 410 # pass it to the user - programming-error here 411 raise 412 413 data = json.dumps(data, indent=2, cls=DjangoJSONEncoder) 414 cache.set(cache_name, data, 60*30) 415 416 return data 417 418 419class ToasterTypeAhead(View): 420 """ A typeahead mechanism to support the front end typeahead widgets """ 421 MAX_RESULTS = 6 422 423 class MissingFieldsException(Exception): 424 pass 425 426 def __init__(self, *args, **kwargs): 427 super(ToasterTypeAhead, self).__init__() 428 429 def get(self, request, *args, **kwargs): 430 def response(data): 431 return HttpResponse(json.dumps(data, 432 indent=2, 433 cls=DjangoJSONEncoder), 434 content_type="application/json") 435 436 error = "ok" 437 438 search_term = request.GET.get("search", None) 439 if search_term is None: 440 # We got no search value so return empty reponse 441 return response({'error': error, 'results': []}) 442 443 try: 444 prj = Project.objects.get(pk=kwargs['pid']) 445 except KeyError: 446 prj = None 447 448 results = self.apply_search(search_term, 449 prj, 450 request)[:ToasterTypeAhead.MAX_RESULTS] 451 452 if len(results) > 0: 453 try: 454 self.validate_fields(results[0]) 455 except self.MissingFieldsException as e: 456 error = e 457 458 data = {'results': results, 459 'error': error} 460 461 return response(data) 462 463 def validate_fields(self, result): 464 if 'name' in result is False or 'detail' in result is False: 465 raise self.MissingFieldsException( 466 "name and detail are required fields") 467 468 def apply_search(self, search_term, prj): 469 """ Override this function to implement search. Return an array of 470 dictionaries with a minium of a name and detail field""" 471 pass 472 473 474class MostRecentBuildsView(View): 475 def _was_yesterday_or_earlier(self, completed_on): 476 now = timezone.now() 477 delta = now - completed_on 478 479 if delta.days >= 1: 480 return True 481 482 return False 483 484 def get(self, request, *args, **kwargs): 485 """ 486 Returns a list of builds in JSON format. 487 """ 488 project = None 489 490 project_id = request.GET.get('project_id', None) 491 if project_id: 492 try: 493 project = Project.objects.get(pk=project_id) 494 except: 495 # if project lookup fails, assume no project 496 pass 497 498 recent_build_objs = Build.get_recent(project) 499 recent_builds = [] 500 501 for build_obj in recent_build_objs: 502 dashboard_url = reverse('builddashboard', args=(build_obj.pk,)) 503 buildtime_url = reverse('buildtime', args=(build_obj.pk,)) 504 rebuild_url = \ 505 reverse('xhr_buildrequest', args=(build_obj.project.pk,)) 506 cancel_url = \ 507 reverse('xhr_buildrequest', args=(build_obj.project.pk,)) 508 509 build = {} 510 build['id'] = build_obj.pk 511 build['dashboard_url'] = dashboard_url 512 513 buildrequest_id = None 514 if hasattr(build_obj, 'buildrequest'): 515 buildrequest_id = build_obj.buildrequest.pk 516 build['buildrequest_id'] = buildrequest_id 517 518 if build_obj.recipes_to_parse > 0: 519 build['recipes_parsed_percentage'] = \ 520 int((build_obj.recipes_parsed / 521 build_obj.recipes_to_parse) * 100) 522 else: 523 build['recipes_parsed_percentage'] = 0 524 if build_obj.repos_to_clone > 0: 525 build['repos_cloned_percentage'] = \ 526 int((build_obj.repos_cloned / 527 build_obj.repos_to_clone) * 100) 528 else: 529 build['repos_cloned_percentage'] = 0 530 531 build['progress_item'] = build_obj.progress_item 532 533 tasks_complete_percentage = 0 534 if build_obj.outcome in (Build.SUCCEEDED, Build.FAILED): 535 tasks_complete_percentage = 100 536 elif build_obj.outcome == Build.IN_PROGRESS: 537 tasks_complete_percentage = build_obj.completeper() 538 build['tasks_complete_percentage'] = tasks_complete_percentage 539 540 build['state'] = build_obj.get_state() 541 542 build['errors'] = build_obj.errors.count() 543 build['dashboard_errors_url'] = dashboard_url + '#errors' 544 545 build['warnings'] = build_obj.warnings.count() 546 build['dashboard_warnings_url'] = dashboard_url + '#warnings' 547 548 build['buildtime'] = sectohms(build_obj.timespent_seconds) 549 build['buildtime_url'] = buildtime_url 550 551 build['rebuild_url'] = rebuild_url 552 build['cancel_url'] = cancel_url 553 554 build['is_default_project_build'] = build_obj.project.is_default 555 556 build['build_targets_json'] = \ 557 template_json(get_tasks(build_obj.target_set.all())) 558 559 # convert completed_on time to user's timezone 560 completed_on = timezone.localtime(build_obj.completed_on) 561 562 completed_on_template = '%H:%M' 563 if self._was_yesterday_or_earlier(completed_on): 564 completed_on_template = '%d/%m/%Y ' + completed_on_template 565 build['completed_on'] = completed_on.strftime( 566 completed_on_template) 567 568 targets = [] 569 target_objs = build_obj.get_sorted_target_list() 570 for target_obj in target_objs: 571 if target_obj.task: 572 targets.append(target_obj.target + ':' + target_obj.task) 573 else: 574 targets.append(target_obj.target) 575 build['targets'] = ' '.join(targets) 576 577 # abbreviated form of the full target list 578 abbreviated_targets = '' 579 num_targets = len(targets) 580 if num_targets > 0: 581 abbreviated_targets = targets[0] 582 if num_targets > 1: 583 abbreviated_targets += (' +%s' % (num_targets - 1)) 584 build['targets_abbreviated'] = abbreviated_targets 585 586 recent_builds.append(build) 587 588 return JsonResponse(recent_builds, safe=False) 589