1#
2# BitBake Toaster Implementation
3#
4# Copyright (C) 2015        Intel Corporation
5#
6# SPDX-License-Identifier: GPL-2.0-only
7#
8
9from django.views.generic import View, TemplateView
10from django.utils.decorators import method_decorator
11from django.views.decorators.cache import cache_control
12from django.shortcuts import HttpResponse
13from django.core.cache import cache
14from django.core.paginator import Paginator, EmptyPage
15from django.db.models import Q
16from orm.models import Project, Build
17from django.template import Context, Template
18from django.template import VariableDoesNotExist
19from django.template import TemplateSyntaxError
20from django.core.serializers.json import DjangoJSONEncoder
21from django.core.exceptions import FieldError
22from django.utils import timezone
23from toastergui.templatetags.projecttags import sectohms, get_tasks
24from toastergui.templatetags.projecttags import json as template_json
25from django.http import JsonResponse
26from django.urls import reverse
27
28import types
29import json
30import collections
31import re
32import os
33
34from toastergui.tablefilter import TableFilterMap
35from toastermain.logs import log_view_mixin
36
37try:
38    from urllib import unquote_plus
39except ImportError:
40    from urllib.parse import unquote_plus
41
42import logging
43logger = logging.getLogger("toaster")
44
45
46class NoFieldOrDataName(Exception):
47    pass
48
49
50class ToasterTable(TemplateView):
51    def __init__(self, *args, **kwargs):
52        super(ToasterTable, self).__init__()
53        if 'template_name' in kwargs:
54            self.template_name = kwargs['template_name']
55        self.title = "Table"
56        self.queryset = None
57        self.columns = []
58
59        # map from field names to Filter instances
60        self.filter_map = TableFilterMap()
61
62        self.total_count = 0
63        self.static_context_extra = {}
64        self.empty_state = "Sorry - no data found"
65        self.default_orderby = ""
66
67    # prevent HTTP caching of table data
68    @method_decorator(cache_control(must_revalidate=True,
69                   max_age=0, no_store=True, no_cache=True))
70    def dispatch(self, *args, **kwargs):
71        return super(ToasterTable, self).dispatch(*args, **kwargs)
72
73    def get_context_data(self, **kwargs):
74        context = super(ToasterTable, self).get_context_data(**kwargs)
75        context['title'] = self.title
76        context['table_name'] = type(self).__name__.lower()
77        context['empty_state'] = self.empty_state
78
79        # global variables
80        context['project_enable'] = ('1' == os.environ.get('TOASTER_BUILDSERVER'))
81        try:
82            context['project_specific'] = ('1' == os.environ.get('TOASTER_PROJECTSPECIFIC'))
83        except:
84            context['project_specific'] = ''
85
86        return context
87
88    @log_view_mixin
89    def get(self, request, *args, **kwargs):
90        if request.GET.get('format', None) == 'json':
91
92            self.setup_queryset(*args, **kwargs)
93            # Put the project id into the context for the static_data_template
94            if 'pid' in kwargs:
95                self.static_context_extra['pid'] = kwargs['pid']
96
97            cmd = request.GET.get('cmd', None)
98            if cmd and 'filterinfo' in cmd:
99                data = self.get_filter_info(request, **kwargs)
100            else:
101                # If no cmd is specified we give you the table data
102                data = self.get_data(request, **kwargs)
103
104            return HttpResponse(data, content_type="application/json")
105
106        return super(ToasterTable, self).get(request, *args, **kwargs)
107
108    def get_filter_info(self, request, **kwargs):
109        self.setup_filters(**kwargs)
110
111        search = request.GET.get("search", None)
112        if search:
113            self.apply_search(search)
114
115        name = request.GET.get("name", None)
116        table_filter = self.filter_map.get_filter(name)
117        return json.dumps(table_filter.to_json(self.queryset),
118                          indent=2,
119                          cls=DjangoJSONEncoder)
120
121    def setup_columns(self, *args, **kwargs):
122        """ function to implement in the subclass which sets up
123        the columns """
124        pass
125
126    def setup_filters(self, *args, **kwargs):
127        """ function to implement in the subclass which sets up the
128        filters """
129        pass
130
131    def setup_queryset(self, *args, **kwargs):
132        """ function to implement in the subclass which sets up the
133        queryset"""
134        pass
135
136    def add_filter(self, table_filter):
137        """Add a filter to the table.
138
139        Args:
140            table_filter: Filter instance
141        """
142        self.filter_map.add_filter(table_filter.name, table_filter)
143
144    def add_column(self, title="", help_text="",
145                   orderable=False, hideable=True, hidden=False,
146                   field_name="", filter_name=None, static_data_name=None,
147                   static_data_template=None):
148        """Add a column to the table.
149
150        Args:
151            title (str): Title for the table header
152            help_text (str): Optional help text to describe the column
153            orderable (bool): Whether the column can be ordered.
154                We order on the field_name.
155            hideable (bool): Whether the user can hide the column
156            hidden (bool): Whether the column is default hidden
157            field_name (str or list): field(s) required for this column's data
158            static_data_name (str, optional): The column's main identifier
159                which will replace the field_name.
160            static_data_template(str, optional): The template to be rendered
161                as data
162        """
163
164        self.columns.append({'title': title,
165                             'help_text': help_text,
166                             'orderable': orderable,
167                             'hideable': hideable,
168                             'hidden': hidden,
169                             'field_name': field_name,
170                             'filter_name': filter_name,
171                             'static_data_name': static_data_name,
172                             'static_data_template': static_data_template})
173
174    def set_column_hidden(self, title, hidden):
175        """
176        Set the hidden state of the column to the value of hidden
177        """
178        for col in self.columns:
179            if col['title'] == title:
180                col['hidden'] = hidden
181                break
182
183    def set_column_hideable(self, title, hideable):
184        """
185        Set the hideable state of the column to the value of hideable
186        """
187        for col in self.columns:
188            if col['title'] == title:
189                col['hideable'] = hideable
190                break
191
192    def render_static_data(self, template, row):
193        """Utility function to render the static data template"""
194
195        context = {
196          'extra': self.static_context_extra,
197          'data': row,
198        }
199
200        context = Context(context)
201        template = Template(template)
202
203        return template.render(context)
204
205    def apply_filter(self, filters, filter_value, **kwargs):
206        """
207        Apply a filter submitted in the querystring to the ToasterTable
208
209        filters: (str) in the format:
210          '<filter name>:<action name>'
211        filter_value: (str) parameters to pass to the named filter
212
213        <filter name> and <action name> are used to look up the correct filter
214        in the ToasterTable's filter map; the <action params> are set on
215        TableFilterAction* before its filter is applied and may modify the
216        queryset returned by the filter
217        """
218        self.setup_filters(**kwargs)
219
220        try:
221            filter_name, action_name = filters.split(':')
222            action_params = unquote_plus(filter_value)
223        except ValueError:
224            return
225
226        if "all" in action_name:
227            return
228
229        try:
230            table_filter = self.filter_map.get_filter(filter_name)
231            action = table_filter.get_action(action_name)
232            action.set_filter_params(action_params)
233            self.queryset = action.filter(self.queryset)
234        except KeyError:
235            # pass it to the user - programming error here
236            raise
237
238    def apply_orderby(self, orderby):
239        # Note that django will execute this when we try to retrieve the data
240        self.queryset = self.queryset.order_by(orderby)
241
242    def apply_search(self, search_term):
243        """Creates a query based on the model's search_allowed_fields"""
244
245        if not hasattr(self.queryset.model, 'search_allowed_fields'):
246            raise Exception("Search fields aren't defined in the model %s"
247                            % self.queryset.model)
248
249        search_queries = None
250        for st in search_term.split(" "):
251            queries = None
252            for field in self.queryset.model.search_allowed_fields:
253                query = Q(**{field + '__icontains': st})
254                if queries:
255                    queries |= query
256                else:
257                    queries = query
258
259            if search_queries:
260                search_queries &= queries
261            else:
262                search_queries = queries
263
264        self.queryset = self.queryset.filter(search_queries)
265
266    def get_data(self, request, **kwargs):
267        """
268        Returns the data for the page requested with the specified
269        parameters applied
270
271        filters: filter and action name, e.g. "outcome:build_succeeded"
272        filter_value: value to pass to the named filter+action, e.g. "on"
273        (for a toggle filter) or "2015-12-11,2015-12-12"
274        (for a date range filter)
275        """
276
277        page_num = request.GET.get("page", 1)
278        limit = request.GET.get("limit", 10)
279        search = request.GET.get("search", None)
280        filters = request.GET.get("filter", None)
281        filter_value = request.GET.get("filter_value", "on")
282        orderby = request.GET.get("orderby", None)
283        nocache = request.GET.get("nocache", None)
284
285        # Make a unique cache name
286        cache_name = self.__class__.__name__
287
288        for key, val in request.GET.items():
289            if key == 'nocache':
290                continue
291            cache_name = cache_name + str(key) + str(val)
292
293        for key, val in kwargs.items():
294            cache_name = cache_name + str(key) + str(val)
295
296        # No special chars allowed in the cache name apart from dash
297        cache_name = re.sub(r'[^A-Za-z0-9-]', "", cache_name)
298
299        if nocache:
300            cache.delete(cache_name)
301
302        data = cache.get(cache_name)
303
304        if data:
305            logger.debug("Got cache data for table '%s'" % self.title)
306            return data
307
308        self.setup_columns(**kwargs)
309
310        self.apply_orderby('pk')
311        if search:
312            self.apply_search(search)
313        if filters:
314            self.apply_filter(filters, filter_value, **kwargs)
315        if orderby:
316            self.apply_orderby(orderby)
317
318        paginator = Paginator(self.queryset, limit)
319
320        try:
321            page = paginator.page(page_num)
322        except EmptyPage:
323            page = paginator.page(1)
324
325        data = {
326            'total': self.queryset.count(),
327            'default_orderby': self.default_orderby,
328            'columns': self.columns,
329            'rows': [],
330            'error': "ok",
331        }
332
333        try:
334            for model_obj in page.object_list:
335                # Use collection to maintain the order
336                required_data = collections.OrderedDict()
337
338                for col in self.columns:
339                    field = col['field_name']
340                    if not field:
341                        field = col['static_data_name']
342                    if not field:
343                        raise NoFieldOrDataName("Must supply a field_name or"
344                                                "static_data_name for column"
345                                                "%s.%s" %
346                                                (self.__class__.__name__, col)
347                                                )
348
349                    # Check if we need to process some static data
350                    if "static_data_name" in col and col['static_data_name']:
351                        # Overwrite the field_name with static_data_name
352                        # so that this can be used as the html class name
353                        col['field_name'] = col['static_data_name']
354
355                        try:
356                            # Render the template given
357                            required_data[col['static_data_name']] = \
358                                    self.render_static_data(
359                                        col['static_data_template'], model_obj)
360                        except (TemplateSyntaxError,
361                                VariableDoesNotExist) as e:
362                            logger.error("could not render template code"
363                                         "%s %s %s",
364                                         col['static_data_template'],
365                                         e, self.__class__.__name__)
366                            required_data[col['static_data_name']] =\
367                                '<!--error-->'
368
369                    else:
370                        # Traverse to any foriegn key in the field
371                        # e.g. recipe__layer_version__name
372                        model_data = None
373
374                        if "__" in field:
375                            for subfield in field.split("__"):
376                                if not model_data:
377                                    # The first iteration is always going to
378                                    # be on the actual model object instance.
379                                    # Subsequent ones are on the result of
380                                    # that. e.g. forieng key objects
381                                    model_data = getattr(model_obj,
382                                                         subfield)
383                                else:
384                                    model_data = getattr(model_data,
385                                                         subfield)
386
387                        else:
388                            model_data = getattr(model_obj,
389                                                 col['field_name'])
390
391                        # We might have a model function as the field so
392                        # call it to return the data needed
393                        if isinstance(model_data, types.MethodType):
394                            model_data = model_data()
395
396                        required_data[col['field_name']] = model_data
397
398                data['rows'].append(required_data)
399
400        except FieldError:
401            # pass  it to the user - programming-error here
402            raise
403
404        data = json.dumps(data, indent=2, cls=DjangoJSONEncoder)
405        cache.set(cache_name, data, 60*30)
406
407        return data
408
409
410class ToasterTypeAhead(View):
411    """ A typeahead mechanism to support the front end typeahead widgets """
412    MAX_RESULTS = 6
413
414    class MissingFieldsException(Exception):
415        pass
416
417    def __init__(self, *args, **kwargs):
418        super(ToasterTypeAhead, self).__init__()
419
420    @log_view_mixin
421    def get(self, request, *args, **kwargs):
422        def response(data):
423            return HttpResponse(json.dumps(data,
424                                           indent=2,
425                                           cls=DjangoJSONEncoder),
426                                content_type="application/json")
427
428        error = "ok"
429
430        search_term = request.GET.get("search", None)
431        if search_term is None:
432            # We got no search value so return empty reponse
433            return response({'error': error, 'results': []})
434
435        try:
436            prj = Project.objects.get(pk=kwargs['pid'])
437        except KeyError:
438            prj = None
439
440        results = self.apply_search(search_term,
441                                    prj,
442                                    request)[:ToasterTypeAhead.MAX_RESULTS]
443
444        if len(results) > 0:
445            try:
446                self.validate_fields(results[0])
447            except self.MissingFieldsException as e:
448                error = e
449
450        data = {'results': results,
451                'error': error}
452
453        return response(data)
454
455    def validate_fields(self, result):
456        if 'name' in result is False or 'detail' in result is False:
457            raise self.MissingFieldsException(
458                "name and detail are required fields")
459
460    def apply_search(self, search_term, prj):
461        """ Override this function to implement search. Return an array of
462        dictionaries with a minium of a name and detail field"""
463        pass
464
465
466class MostRecentBuildsView(View):
467    def _was_yesterday_or_earlier(self, completed_on):
468        now = timezone.now()
469        delta = now - completed_on
470
471        if delta.days >= 1:
472            return True
473
474        return False
475
476    @log_view_mixin
477    def get(self, request, *args, **kwargs):
478        """
479        Returns a list of builds in JSON format.
480        """
481        project = None
482
483        project_id = request.GET.get('project_id', None)
484        if project_id:
485            try:
486                project = Project.objects.get(pk=project_id)
487            except:
488                # if project lookup fails, assume no project
489                pass
490
491        recent_build_objs = Build.get_recent(project)
492        recent_builds = []
493
494        for build_obj in recent_build_objs:
495            dashboard_url = reverse('builddashboard', args=(build_obj.pk,))
496            buildtime_url = reverse('buildtime', args=(build_obj.pk,))
497            rebuild_url = \
498                reverse('xhr_buildrequest', args=(build_obj.project.pk,))
499            cancel_url = \
500                reverse('xhr_buildrequest', args=(build_obj.project.pk,))
501
502            build = {}
503            build['id'] = build_obj.pk
504            build['dashboard_url'] = dashboard_url
505
506            buildrequest_id = None
507            if hasattr(build_obj, 'buildrequest'):
508                buildrequest_id = build_obj.buildrequest.pk
509            build['buildrequest_id'] = buildrequest_id
510
511            if build_obj.recipes_to_parse > 0:
512                build['recipes_parsed_percentage'] = \
513                    int((build_obj.recipes_parsed /
514                         build_obj.recipes_to_parse) * 100)
515            else:
516                build['recipes_parsed_percentage'] = 0
517            if build_obj.repos_to_clone > 0:
518                build['repos_cloned_percentage'] = \
519                    int((build_obj.repos_cloned /
520                         build_obj.repos_to_clone) * 100)
521            else:
522                build['repos_cloned_percentage'] = 0
523
524            build['progress_item'] = build_obj.progress_item
525
526            tasks_complete_percentage = 0
527            if build_obj.outcome in (Build.SUCCEEDED, Build.FAILED):
528                tasks_complete_percentage = 100
529            elif build_obj.outcome == Build.IN_PROGRESS:
530                tasks_complete_percentage = build_obj.completeper()
531            build['tasks_complete_percentage'] = tasks_complete_percentage
532
533            build['state'] = build_obj.get_state()
534
535            build['errors'] = build_obj.errors.count()
536            build['dashboard_errors_url'] = dashboard_url + '#errors'
537
538            build['warnings'] = build_obj.warnings.count()
539            build['dashboard_warnings_url'] = dashboard_url + '#warnings'
540
541            build['buildtime'] = sectohms(build_obj.timespent_seconds)
542            build['buildtime_url'] = buildtime_url
543
544            build['rebuild_url'] = rebuild_url
545            build['cancel_url'] = cancel_url
546
547            build['is_default_project_build'] = build_obj.project.is_default
548
549            build['build_targets_json'] = \
550                template_json(get_tasks(build_obj.target_set.all()))
551
552            # convert completed_on time to user's timezone
553            completed_on = timezone.localtime(build_obj.completed_on)
554
555            completed_on_template = '%H:%M'
556            if self._was_yesterday_or_earlier(completed_on):
557                completed_on_template = '%d/%m/%Y ' + completed_on_template
558            build['completed_on'] = completed_on.strftime(
559                completed_on_template)
560
561            targets = []
562            target_objs = build_obj.get_sorted_target_list()
563            for target_obj in target_objs:
564                if target_obj.task:
565                    targets.append(target_obj.target + ':' + target_obj.task)
566                else:
567                    targets.append(target_obj.target)
568            build['targets'] = ' '.join(targets)
569
570            # abbreviated form of the full target list
571            abbreviated_targets = ''
572            num_targets = len(targets)
573            if num_targets > 0:
574                abbreviated_targets = targets[0]
575            if num_targets > 1:
576                abbreviated_targets += (' +%s' % (num_targets - 1))
577            build['targets_abbreviated'] = abbreviated_targets
578
579            recent_builds.append(build)
580
581        return JsonResponse(recent_builds, safe=False)
582