1# Class for actually running tests.
2#
3# Copyright (c) 2020-2021 Virtuozzo International GmbH
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation; either version 2 of the License, or
8# (at your option) any later version.
9#
10# This program is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License
16# along with this program.  If not, see <http://www.gnu.org/licenses/>.
17#
18
19import os
20from pathlib import Path
21import datetime
22import time
23import difflib
24import subprocess
25import contextlib
26import json
27import termios
28import sys
29from multiprocessing import Pool
30from contextlib import contextmanager
31from typing import List, Optional, Iterator, Any, Sequence, Dict, \
32        ContextManager
33
34from testenv import TestEnv
35
36
37def silent_unlink(path: Path) -> None:
38    try:
39        path.unlink()
40    except OSError:
41        pass
42
43
44def file_diff(file1: str, file2: str) -> List[str]:
45    with open(file1, encoding="utf-8") as f1, \
46         open(file2, encoding="utf-8") as f2:
47        # We want to ignore spaces at line ends. There are a lot of mess about
48        # it in iotests.
49        # TODO: fix all tests to not produce extra spaces, fix all .out files
50        # and use strict diff here!
51        seq1 = [line.rstrip() for line in f1]
52        seq2 = [line.rstrip() for line in f2]
53        res = [line.rstrip()
54               for line in difflib.unified_diff(seq1, seq2, file1, file2)]
55        return res
56
57
58# We want to save current tty settings during test run,
59# since an aborting qemu call may leave things screwed up.
60@contextmanager
61def savetty() -> Iterator[None]:
62    isterm = sys.stdin.isatty()
63    if isterm:
64        fd = sys.stdin.fileno()
65        attr = termios.tcgetattr(fd)
66
67    try:
68        yield
69    finally:
70        if isterm:
71            termios.tcsetattr(fd, termios.TCSADRAIN, attr)
72
73
74class LastElapsedTime(ContextManager['LastElapsedTime']):
75    """ Cache for elapsed time for tests, to show it during new test run
76
77    It is safe to use get() at any time.  To use update(), you must either
78    use it inside with-block or use save() after update().
79    """
80    def __init__(self, cache_file: str, env: TestEnv) -> None:
81        self.env = env
82        self.cache_file = cache_file
83        self.cache: Dict[str, Dict[str, Dict[str, float]]]
84
85        try:
86            with open(cache_file, encoding="utf-8") as f:
87                self.cache = json.load(f)
88        except (OSError, ValueError):
89            self.cache = {}
90
91    def get(self, test: str,
92            default: Optional[float] = None) -> Optional[float]:
93        if test not in self.cache:
94            return default
95
96        if self.env.imgproto not in self.cache[test]:
97            return default
98
99        return self.cache[test][self.env.imgproto].get(self.env.imgfmt,
100                                                       default)
101
102    def update(self, test: str, elapsed: float) -> None:
103        d = self.cache.setdefault(test, {})
104        d.setdefault(self.env.imgproto, {})[self.env.imgfmt] = elapsed
105
106    def save(self) -> None:
107        with open(self.cache_file, 'w', encoding="utf-8") as f:
108            json.dump(self.cache, f)
109
110    def __enter__(self) -> 'LastElapsedTime':
111        return self
112
113    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
114        self.save()
115
116
117class TestResult:
118    def __init__(self, status: str, description: str = '',
119                 elapsed: Optional[float] = None, diff: Sequence[str] = (),
120                 casenotrun: str = '', interrupted: bool = False) -> None:
121        self.status = status
122        self.description = description
123        self.elapsed = elapsed
124        self.diff = diff
125        self.casenotrun = casenotrun
126        self.interrupted = interrupted
127
128
129class TestRunner(ContextManager['TestRunner']):
130    shared_self = None
131
132    @staticmethod
133    def proc_run_test(test: str, test_field_width: int) -> TestResult:
134        # We are in a subprocess, we can't change the runner object!
135        runner = TestRunner.shared_self
136        assert runner is not None
137        return runner.run_test(test, test_field_width, mp=True)
138
139    def run_tests_pool(self, tests: List[str],
140                       test_field_width: int, jobs: int) -> List[TestResult]:
141
142        # passing self directly to Pool.starmap() just doesn't work, because
143        # it's a context manager.
144        assert TestRunner.shared_self is None
145        TestRunner.shared_self = self
146
147        with Pool(jobs) as p:
148            results = p.starmap(self.proc_run_test,
149                                zip(tests, [test_field_width] * len(tests)))
150
151        TestRunner.shared_self = None
152
153        return results
154
155    def __init__(self, env: TestEnv, makecheck: bool = False,
156                 color: str = 'auto') -> None:
157        self.env = env
158        self.makecheck = makecheck
159        self.last_elapsed = LastElapsedTime('.last-elapsed-cache', env)
160
161        assert color in ('auto', 'on', 'off')
162        self.color = (color == 'on') or (color == 'auto' and
163                                         sys.stdout.isatty())
164
165        self._stack: contextlib.ExitStack
166
167    def __enter__(self) -> 'TestRunner':
168        self._stack = contextlib.ExitStack()
169        self._stack.enter_context(self.env)
170        self._stack.enter_context(self.last_elapsed)
171        self._stack.enter_context(savetty())
172        return self
173
174    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
175        self._stack.close()
176
177    def test_print_one_line(self, test: str, starttime: str,
178                            endtime: Optional[str] = None, status: str = '...',
179                            lasttime: Optional[float] = None,
180                            thistime: Optional[float] = None,
181                            description: str = '',
182                            test_field_width: Optional[int] = None,
183                            end: str = '\n') -> None:
184        """ Print short test info before/after test run """
185        test = os.path.basename(test)
186
187        if test_field_width is None:
188            test_field_width = 8
189
190        if self.makecheck and status != '...':
191            if status and status != 'pass':
192                status = f' [{status}]'
193            else:
194                status = ''
195
196            print(f'  TEST   iotest-{self.env.imgfmt}: {test}{status}')
197            return
198
199        if lasttime:
200            lasttime_s = f' (last: {lasttime:.1f}s)'
201        else:
202            lasttime_s = ''
203        if thistime:
204            thistime_s = f'{thistime:.1f}s'
205        else:
206            thistime_s = '...'
207
208        if endtime:
209            endtime = f'[{endtime}]'
210        else:
211            endtime = ''
212
213        if self.color:
214            if status == 'pass':
215                col = '\033[32m'
216            elif status == 'fail':
217                col = '\033[1m\033[31m'
218            elif status == 'not run':
219                col = '\033[33m'
220            else:
221                col = ''
222
223            col_end = '\033[0m'
224        else:
225            col = ''
226            col_end = ''
227
228        print(f'{test:{test_field_width}} {col}{status:10}{col_end} '
229              f'[{starttime}] {endtime:13}{thistime_s:5} {lasttime_s:14} '
230              f'{description}', end=end)
231
232    def find_reference(self, test: str) -> str:
233        if self.env.cachemode == 'none':
234            ref = f'{test}.out.nocache'
235            if os.path.isfile(ref):
236                return ref
237
238        ref = f'{test}.out.{self.env.imgfmt}'
239        if os.path.isfile(ref):
240            return ref
241
242        ref = f'{test}.{self.env.qemu_default_machine}.out'
243        if os.path.isfile(ref):
244            return ref
245
246        return f'{test}.out'
247
248    def do_run_test(self, test: str, mp: bool) -> TestResult:
249        """
250        Run one test
251
252        :param test: test file path
253        :param mp: if true, we are in a multiprocessing environment, use
254                   personal subdirectories for test run
255
256        Note: this method may be called from subprocess, so it does not
257        change ``self`` object in any way!
258        """
259
260        f_test = Path(test)
261        f_bad = Path(f_test.name + '.out.bad')
262        f_notrun = Path(f_test.name + '.notrun')
263        f_casenotrun = Path(f_test.name + '.casenotrun')
264        f_reference = Path(self.find_reference(test))
265
266        if not f_test.exists():
267            return TestResult(status='fail',
268                              description=f'No such test file: {f_test}')
269
270        if not os.access(str(f_test), os.X_OK):
271            sys.exit(f'Not executable: {f_test}')
272
273        if not f_reference.exists():
274            return TestResult(status='not run',
275                              description='No qualified output '
276                                          f'(expected {f_reference})')
277
278        for p in (f_bad, f_notrun, f_casenotrun):
279            silent_unlink(p)
280
281        args = [str(f_test.resolve())]
282        env = self.env.prepare_subprocess(args)
283        if mp:
284            # Split test directories, so that tests running in parallel don't
285            # break each other.
286            for d in ['TEST_DIR', 'SOCK_DIR']:
287                env[d] = os.path.join(env[d], f_test.name)
288                Path(env[d]).mkdir(parents=True, exist_ok=True)
289
290        t0 = time.time()
291        with f_bad.open('w', encoding="utf-8") as f:
292            with subprocess.Popen(args, cwd=str(f_test.parent), env=env,
293                                  stdout=f, stderr=subprocess.STDOUT) as proc:
294                try:
295                    proc.wait()
296                except KeyboardInterrupt:
297                    proc.terminate()
298                    proc.wait()
299                    return TestResult(status='not run',
300                                      description='Interrupted by user',
301                                      interrupted=True)
302                ret = proc.returncode
303
304        elapsed = round(time.time() - t0, 1)
305
306        if ret != 0:
307            return TestResult(status='fail', elapsed=elapsed,
308                              description=f'failed, exit status {ret}',
309                              diff=file_diff(str(f_reference), str(f_bad)))
310
311        if f_notrun.exists():
312            return TestResult(
313                status='not run',
314                description=f_notrun.read_text(encoding='utf-8').strip())
315
316        casenotrun = ''
317        if f_casenotrun.exists():
318            casenotrun = f_casenotrun.read_text(encoding='utf-8')
319
320        diff = file_diff(str(f_reference), str(f_bad))
321        if diff:
322            return TestResult(status='fail', elapsed=elapsed,
323                              description=f'output mismatch (see {f_bad})',
324                              diff=diff, casenotrun=casenotrun)
325        else:
326            f_bad.unlink()
327            return TestResult(status='pass', elapsed=elapsed,
328                              casenotrun=casenotrun)
329
330    def run_test(self, test: str,
331                 test_field_width: Optional[int] = None,
332                 mp: bool = False) -> TestResult:
333        """
334        Run one test and print short status
335
336        :param test: test file path
337        :param test_field_width: width for first field of status format
338        :param mp: if true, we are in a multiprocessing environment, don't try
339                   to rewrite things in stdout
340
341        Note: this method may be called from subprocess, so it does not
342        change ``self`` object in any way!
343        """
344
345        last_el = self.last_elapsed.get(test)
346        start = datetime.datetime.now().strftime('%H:%M:%S')
347
348        if not self.makecheck:
349            self.test_print_one_line(test=test,
350                                     status = 'started' if mp else '...',
351                                     starttime=start,
352                                     lasttime=last_el,
353                                     end = '\n' if mp else '\r',
354                                     test_field_width=test_field_width)
355
356        res = self.do_run_test(test, mp)
357
358        end = datetime.datetime.now().strftime('%H:%M:%S')
359        self.test_print_one_line(test=test, status=res.status,
360                                 starttime=start, endtime=end,
361                                 lasttime=last_el, thistime=res.elapsed,
362                                 description=res.description,
363                                 test_field_width=test_field_width)
364
365        if res.casenotrun:
366            print(res.casenotrun)
367
368        return res
369
370    def run_tests(self, tests: List[str], jobs: int = 1) -> bool:
371        n_run = 0
372        failed = []
373        notrun = []
374        casenotrun = []
375
376        if not self.makecheck:
377            self.env.print_env()
378
379        test_field_width = max(len(os.path.basename(t)) for t in tests) + 2
380
381        if jobs > 1:
382            results = self.run_tests_pool(tests, test_field_width, jobs)
383
384        for i, t in enumerate(tests):
385            name = os.path.basename(t)
386
387            if jobs > 1:
388                res = results[i]
389            else:
390                res = self.run_test(t, test_field_width)
391
392            assert res.status in ('pass', 'fail', 'not run')
393
394            if res.casenotrun:
395                casenotrun.append(t)
396
397            if res.status != 'not run':
398                n_run += 1
399
400            if res.status == 'fail':
401                failed.append(name)
402                if self.makecheck:
403                    self.env.print_env()
404                if res.diff:
405                    print('\n'.join(res.diff))
406            elif res.status == 'not run':
407                notrun.append(name)
408            elif res.status == 'pass':
409                assert res.elapsed is not None
410                self.last_elapsed.update(t, res.elapsed)
411
412            sys.stdout.flush()
413            if res.interrupted:
414                break
415
416        if notrun:
417            print('Not run:', ' '.join(notrun))
418
419        if casenotrun:
420            print('Some cases not run in:', ' '.join(casenotrun))
421
422        if failed:
423            print('Failures:', ' '.join(failed))
424            print(f'Failed {len(failed)} of {n_run} iotests')
425            return False
426        else:
427            print(f'Passed all {n_run} iotests')
428            return True
429