1#
2# ex:ts=4:sw=4:sts=4:et
3# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4#
5# BitBake Toaster Implementation
6#
7# Copyright (C) 2015        Intel Corporation
8#
9# This program is free software; you can redistribute it and/or modify
10# it under the terms of the GNU General Public License version 2 as
11# published by the Free Software Foundation.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the GNU General Public License along
19# with this program; if not, write to the Free Software Foundation, Inc.,
20# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21
22from django.views.generic import View, TemplateView
23from django.views.decorators.cache import cache_control
24from django.shortcuts import HttpResponse
25from django.core.cache import cache
26from django.core.paginator import Paginator, EmptyPage
27from django.db.models import Q
28from orm.models import Project, Build
29from django.template import Context, Template
30from django.template import VariableDoesNotExist
31from django.template import TemplateSyntaxError
32from django.core.serializers.json import DjangoJSONEncoder
33from django.core.exceptions import FieldError
34from django.utils import timezone
35from toastergui.templatetags.projecttags import sectohms, get_tasks
36from toastergui.templatetags.projecttags import json as template_json
37from django.http import JsonResponse
38from django.core.urlresolvers import reverse
39
40import types
41import json
42import collections
43import re
44import os
45
46from toastergui.tablefilter import TableFilterMap
47
48try:
49    from urllib import unquote_plus
50except ImportError:
51    from urllib.parse import unquote_plus
52
53import logging
54logger = logging.getLogger("toaster")
55
56
57class NoFieldOrDataName(Exception):
58    pass
59
60
61class ToasterTable(TemplateView):
62    def __init__(self, *args, **kwargs):
63        super(ToasterTable, self).__init__()
64        if 'template_name' in kwargs:
65            self.template_name = kwargs['template_name']
66        self.title = "Table"
67        self.queryset = None
68        self.columns = []
69
70        # map from field names to Filter instances
71        self.filter_map = TableFilterMap()
72
73        self.total_count = 0
74        self.static_context_extra = {}
75        self.empty_state = "Sorry - no data found"
76        self.default_orderby = ""
77
78    # prevent HTTP caching of table data
79    @cache_control(must_revalidate=True,
80                   max_age=0, no_store=True, no_cache=True)
81    def dispatch(self, *args, **kwargs):
82        return super(ToasterTable, self).dispatch(*args, **kwargs)
83
84    def get_context_data(self, **kwargs):
85        context = super(ToasterTable, self).get_context_data(**kwargs)
86        context['title'] = self.title
87        context['table_name'] = type(self).__name__.lower()
88        context['empty_state'] = self.empty_state
89
90        # global variables
91        context['project_enable'] = ('1' == os.environ.get('TOASTER_BUILDSERVER'))
92        try:
93            context['project_specific'] = ('1' == os.environ.get('TOASTER_PROJECTSPECIFIC'))
94        except:
95            context['project_specific'] = ''
96
97        return context
98
99    def get(self, request, *args, **kwargs):
100        if request.GET.get('format', None) == 'json':
101
102            self.setup_queryset(*args, **kwargs)
103            # Put the project id into the context for the static_data_template
104            if 'pid' in kwargs:
105                self.static_context_extra['pid'] = kwargs['pid']
106
107            cmd = request.GET.get('cmd', None)
108            if cmd and 'filterinfo' in cmd:
109                data = self.get_filter_info(request, **kwargs)
110            else:
111                # If no cmd is specified we give you the table data
112                data = self.get_data(request, **kwargs)
113
114            return HttpResponse(data, content_type="application/json")
115
116        return super(ToasterTable, self).get(request, *args, **kwargs)
117
118    def get_filter_info(self, request, **kwargs):
119        self.setup_filters(**kwargs)
120
121        search = request.GET.get("search", None)
122        if search:
123            self.apply_search(search)
124
125        name = request.GET.get("name", None)
126        table_filter = self.filter_map.get_filter(name)
127        return json.dumps(table_filter.to_json(self.queryset),
128                          indent=2,
129                          cls=DjangoJSONEncoder)
130
131    def setup_columns(self, *args, **kwargs):
132        """ function to implement in the subclass which sets up
133        the columns """
134        pass
135
136    def setup_filters(self, *args, **kwargs):
137        """ function to implement in the subclass which sets up the
138        filters """
139        pass
140
141    def setup_queryset(self, *args, **kwargs):
142        """ function to implement in the subclass which sets up the
143        queryset"""
144        pass
145
146    def add_filter(self, table_filter):
147        """Add a filter to the table.
148
149        Args:
150            table_filter: Filter instance
151        """
152        self.filter_map.add_filter(table_filter.name, table_filter)
153
154    def add_column(self, title="", help_text="",
155                   orderable=False, hideable=True, hidden=False,
156                   field_name="", filter_name=None, static_data_name=None,
157                   static_data_template=None):
158        """Add a column to the table.
159
160        Args:
161            title (str): Title for the table header
162            help_text (str): Optional help text to describe the column
163            orderable (bool): Whether the column can be ordered.
164                We order on the field_name.
165            hideable (bool): Whether the user can hide the column
166            hidden (bool): Whether the column is default hidden
167            field_name (str or list): field(s) required for this column's data
168            static_data_name (str, optional): The column's main identifier
169                which will replace the field_name.
170            static_data_template(str, optional): The template to be rendered
171                as data
172        """
173
174        self.columns.append({'title': title,
175                             'help_text': help_text,
176                             'orderable': orderable,
177                             'hideable': hideable,
178                             'hidden': hidden,
179                             'field_name': field_name,
180                             'filter_name': filter_name,
181                             'static_data_name': static_data_name,
182                             'static_data_template': static_data_template})
183
184    def set_column_hidden(self, title, hidden):
185        """
186        Set the hidden state of the column to the value of hidden
187        """
188        for col in self.columns:
189            if col['title'] == title:
190                col['hidden'] = hidden
191                break
192
193    def set_column_hideable(self, title, hideable):
194        """
195        Set the hideable state of the column to the value of hideable
196        """
197        for col in self.columns:
198            if col['title'] == title:
199                col['hideable'] = hideable
200                break
201
202    def render_static_data(self, template, row):
203        """Utility function to render the static data template"""
204
205        context = {
206          'extra': self.static_context_extra,
207          'data': row,
208        }
209
210        context = Context(context)
211        template = Template(template)
212
213        return template.render(context)
214
215    def apply_filter(self, filters, filter_value, **kwargs):
216        """
217        Apply a filter submitted in the querystring to the ToasterTable
218
219        filters: (str) in the format:
220          '<filter name>:<action name>'
221        filter_value: (str) parameters to pass to the named filter
222
223        <filter name> and <action name> are used to look up the correct filter
224        in the ToasterTable's filter map; the <action params> are set on
225        TableFilterAction* before its filter is applied and may modify the
226        queryset returned by the filter
227        """
228        self.setup_filters(**kwargs)
229
230        try:
231            filter_name, action_name = filters.split(':')
232            action_params = unquote_plus(filter_value)
233        except ValueError:
234            return
235
236        if "all" in action_name:
237            return
238
239        try:
240            table_filter = self.filter_map.get_filter(filter_name)
241            action = table_filter.get_action(action_name)
242            action.set_filter_params(action_params)
243            self.queryset = action.filter(self.queryset)
244        except KeyError:
245            # pass it to the user - programming error here
246            raise
247
248    def apply_orderby(self, orderby):
249        # Note that django will execute this when we try to retrieve the data
250        self.queryset = self.queryset.order_by(orderby)
251
252    def apply_search(self, search_term):
253        """Creates a query based on the model's search_allowed_fields"""
254
255        if not hasattr(self.queryset.model, 'search_allowed_fields'):
256            raise Exception("Search fields aren't defined in the model %s"
257                            % self.queryset.model)
258
259        search_queries = None
260        for st in search_term.split(" "):
261            queries = None
262            for field in self.queryset.model.search_allowed_fields:
263                query = Q(**{field + '__icontains': st})
264                if queries:
265                    queries |= query
266                else:
267                    queries = query
268
269            if search_queries:
270                search_queries &= queries
271            else:
272                search_queries = queries
273
274        self.queryset = self.queryset.filter(search_queries)
275
276    def get_data(self, request, **kwargs):
277        """
278        Returns the data for the page requested with the specified
279        parameters applied
280
281        filters: filter and action name, e.g. "outcome:build_succeeded"
282        filter_value: value to pass to the named filter+action, e.g. "on"
283        (for a toggle filter) or "2015-12-11,2015-12-12"
284        (for a date range filter)
285        """
286
287        page_num = request.GET.get("page", 1)
288        limit = request.GET.get("limit", 10)
289        search = request.GET.get("search", None)
290        filters = request.GET.get("filter", None)
291        filter_value = request.GET.get("filter_value", "on")
292        orderby = request.GET.get("orderby", None)
293        nocache = request.GET.get("nocache", None)
294
295        # Make a unique cache name
296        cache_name = self.__class__.__name__
297
298        for key, val in request.GET.items():
299            if key == 'nocache':
300                continue
301            cache_name = cache_name + str(key) + str(val)
302
303        for key, val in kwargs.items():
304            cache_name = cache_name + str(key) + str(val)
305
306        # No special chars allowed in the cache name apart from dash
307        cache_name = re.sub(r'[^A-Za-z0-9-]', "", cache_name)
308
309        if nocache:
310            cache.delete(cache_name)
311
312        data = cache.get(cache_name)
313
314        if data:
315            logger.debug("Got cache data for table '%s'" % self.title)
316            return data
317
318        self.setup_columns(**kwargs)
319
320        if search:
321            self.apply_search(search)
322        if filters:
323            self.apply_filter(filters, filter_value, **kwargs)
324        if orderby:
325            self.apply_orderby(orderby)
326
327        paginator = Paginator(self.queryset, limit)
328
329        try:
330            page = paginator.page(page_num)
331        except EmptyPage:
332            page = paginator.page(1)
333
334        data = {
335            'total': self.queryset.count(),
336            'default_orderby': self.default_orderby,
337            'columns': self.columns,
338            'rows': [],
339            'error': "ok",
340        }
341
342        try:
343            for model_obj in page.object_list:
344                # Use collection to maintain the order
345                required_data = collections.OrderedDict()
346
347                for col in self.columns:
348                    field = col['field_name']
349                    if not field:
350                        field = col['static_data_name']
351                    if not field:
352                        raise NoFieldOrDataName("Must supply a field_name or"
353                                                "static_data_name for column"
354                                                "%s.%s" %
355                                                (self.__class__.__name__, col)
356                                                )
357
358                    # Check if we need to process some static data
359                    if "static_data_name" in col and col['static_data_name']:
360                        # Overwrite the field_name with static_data_name
361                        # so that this can be used as the html class name
362                        col['field_name'] = col['static_data_name']
363
364                        try:
365                            # Render the template given
366                            required_data[col['static_data_name']] = \
367                                    self.render_static_data(
368                                        col['static_data_template'], model_obj)
369                        except (TemplateSyntaxError,
370                                VariableDoesNotExist) as e:
371                            logger.error("could not render template code"
372                                         "%s %s %s",
373                                         col['static_data_template'],
374                                         e, self.__class__.__name__)
375                            required_data[col['static_data_name']] =\
376                                '<!--error-->'
377
378                    else:
379                        # Traverse to any foriegn key in the field
380                        # e.g. recipe__layer_version__name
381                        model_data = None
382
383                        if "__" in field:
384                            for subfield in field.split("__"):
385                                if not model_data:
386                                    # The first iteration is always going to
387                                    # be on the actual model object instance.
388                                    # Subsequent ones are on the result of
389                                    # that. e.g. forieng key objects
390                                    model_data = getattr(model_obj,
391                                                         subfield)
392                                else:
393                                    model_data = getattr(model_data,
394                                                         subfield)
395
396                        else:
397                            model_data = getattr(model_obj,
398                                                 col['field_name'])
399
400                        # We might have a model function as the field so
401                        # call it to return the data needed
402                        if isinstance(model_data, types.MethodType):
403                            model_data = model_data()
404
405                        required_data[col['field_name']] = model_data
406
407                data['rows'].append(required_data)
408
409        except FieldError:
410            # pass  it to the user - programming-error here
411            raise
412
413        data = json.dumps(data, indent=2, cls=DjangoJSONEncoder)
414        cache.set(cache_name, data, 60*30)
415
416        return data
417
418
419class ToasterTypeAhead(View):
420    """ A typeahead mechanism to support the front end typeahead widgets """
421    MAX_RESULTS = 6
422
423    class MissingFieldsException(Exception):
424        pass
425
426    def __init__(self, *args, **kwargs):
427        super(ToasterTypeAhead, self).__init__()
428
429    def get(self, request, *args, **kwargs):
430        def response(data):
431            return HttpResponse(json.dumps(data,
432                                           indent=2,
433                                           cls=DjangoJSONEncoder),
434                                content_type="application/json")
435
436        error = "ok"
437
438        search_term = request.GET.get("search", None)
439        if search_term is None:
440            # We got no search value so return empty reponse
441            return response({'error': error, 'results': []})
442
443        try:
444            prj = Project.objects.get(pk=kwargs['pid'])
445        except KeyError:
446            prj = None
447
448        results = self.apply_search(search_term,
449                                    prj,
450                                    request)[:ToasterTypeAhead.MAX_RESULTS]
451
452        if len(results) > 0:
453            try:
454                self.validate_fields(results[0])
455            except self.MissingFieldsException as e:
456                error = e
457
458        data = {'results': results,
459                'error': error}
460
461        return response(data)
462
463    def validate_fields(self, result):
464        if 'name' in result is False or 'detail' in result is False:
465            raise self.MissingFieldsException(
466                "name and detail are required fields")
467
468    def apply_search(self, search_term, prj):
469        """ Override this function to implement search. Return an array of
470        dictionaries with a minium of a name and detail field"""
471        pass
472
473
474class MostRecentBuildsView(View):
475    def _was_yesterday_or_earlier(self, completed_on):
476        now = timezone.now()
477        delta = now - completed_on
478
479        if delta.days >= 1:
480            return True
481
482        return False
483
484    def get(self, request, *args, **kwargs):
485        """
486        Returns a list of builds in JSON format.
487        """
488        project = None
489
490        project_id = request.GET.get('project_id', None)
491        if project_id:
492            try:
493                project = Project.objects.get(pk=project_id)
494            except:
495                # if project lookup fails, assume no project
496                pass
497
498        recent_build_objs = Build.get_recent(project)
499        recent_builds = []
500
501        for build_obj in recent_build_objs:
502            dashboard_url = reverse('builddashboard', args=(build_obj.pk,))
503            buildtime_url = reverse('buildtime', args=(build_obj.pk,))
504            rebuild_url = \
505                reverse('xhr_buildrequest', args=(build_obj.project.pk,))
506            cancel_url = \
507                reverse('xhr_buildrequest', args=(build_obj.project.pk,))
508
509            build = {}
510            build['id'] = build_obj.pk
511            build['dashboard_url'] = dashboard_url
512
513            buildrequest_id = None
514            if hasattr(build_obj, 'buildrequest'):
515                buildrequest_id = build_obj.buildrequest.pk
516            build['buildrequest_id'] = buildrequest_id
517
518            if build_obj.recipes_to_parse > 0:
519                build['recipes_parsed_percentage'] = \
520                    int((build_obj.recipes_parsed /
521                         build_obj.recipes_to_parse) * 100)
522            else:
523                build['recipes_parsed_percentage'] = 0
524            if build_obj.repos_to_clone > 0:
525                build['repos_cloned_percentage'] = \
526                    int((build_obj.repos_cloned /
527                         build_obj.repos_to_clone) * 100)
528            else:
529                build['repos_cloned_percentage'] = 0
530
531            build['progress_item'] = build_obj.progress_item
532
533            tasks_complete_percentage = 0
534            if build_obj.outcome in (Build.SUCCEEDED, Build.FAILED):
535                tasks_complete_percentage = 100
536            elif build_obj.outcome == Build.IN_PROGRESS:
537                tasks_complete_percentage = build_obj.completeper()
538            build['tasks_complete_percentage'] = tasks_complete_percentage
539
540            build['state'] = build_obj.get_state()
541
542            build['errors'] = build_obj.errors.count()
543            build['dashboard_errors_url'] = dashboard_url + '#errors'
544
545            build['warnings'] = build_obj.warnings.count()
546            build['dashboard_warnings_url'] = dashboard_url + '#warnings'
547
548            build['buildtime'] = sectohms(build_obj.timespent_seconds)
549            build['buildtime_url'] = buildtime_url
550
551            build['rebuild_url'] = rebuild_url
552            build['cancel_url'] = cancel_url
553
554            build['is_default_project_build'] = build_obj.project.is_default
555
556            build['build_targets_json'] = \
557                template_json(get_tasks(build_obj.target_set.all()))
558
559            # convert completed_on time to user's timezone
560            completed_on = timezone.localtime(build_obj.completed_on)
561
562            completed_on_template = '%H:%M'
563            if self._was_yesterday_or_earlier(completed_on):
564                completed_on_template = '%d/%m/%Y ' + completed_on_template
565            build['completed_on'] = completed_on.strftime(
566                completed_on_template)
567
568            targets = []
569            target_objs = build_obj.get_sorted_target_list()
570            for target_obj in target_objs:
571                if target_obj.task:
572                    targets.append(target_obj.target + ':' + target_obj.task)
573                else:
574                    targets.append(target_obj.target)
575            build['targets'] = ' '.join(targets)
576
577            # abbreviated form of the full target list
578            abbreviated_targets = ''
579            num_targets = len(targets)
580            if num_targets > 0:
581                abbreviated_targets = targets[0]
582            if num_targets > 1:
583                abbreviated_targets += (' +%s' % (num_targets - 1))
584            build['targets_abbreviated'] = abbreviated_targets
585
586            recent_builds.append(build)
587
588        return JsonResponse(recent_builds, safe=False)
589