1903cb1bfSPhilippe Mathieu-Daudé#!/usr/bin/env python3 2*9dd003a9SVladimir Sementsov-Ogievskiy# group: rw 34214faceSAlberto Garcia# 44214faceSAlberto Garcia# Tests for block device statistics 54214faceSAlberto Garcia# 64214faceSAlberto Garcia# Copyright (C) 2015 Igalia, S.L. 74214faceSAlberto Garcia# Author: Alberto Garcia <berto@igalia.com> 84214faceSAlberto Garcia# 94214faceSAlberto Garcia# This program is free software; you can redistribute it and/or modify 104214faceSAlberto Garcia# it under the terms of the GNU General Public License as published by 114214faceSAlberto Garcia# the Free Software Foundation; either version 2 of the License, or 124214faceSAlberto Garcia# (at your option) any later version. 134214faceSAlberto Garcia# 144214faceSAlberto Garcia# This program is distributed in the hope that it will be useful, 154214faceSAlberto Garcia# but WITHOUT ANY WARRANTY; without even the implied warranty of 164214faceSAlberto Garcia# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 174214faceSAlberto Garcia# GNU General Public License for more details. 184214faceSAlberto Garcia# 194214faceSAlberto Garcia# You should have received a copy of the GNU General Public License 204214faceSAlberto Garcia# along with this program. If not, see <http://www.gnu.org/licenses/>. 214214faceSAlberto Garcia# 224214faceSAlberto Garcia 234214faceSAlberto Garciaimport iotests 244214faceSAlberto Garciaimport os 254214faceSAlberto Garcia 264214faceSAlberto Garciainterval_length = 10 274214faceSAlberto Garciansec_per_sec = 1000000000 289a3a9a63SMax Reitzop_latency = nsec_per_sec // 1000 # See qtest_latency_ns in accounting.c 294214faceSAlberto Garciabad_sector = 8192 304214faceSAlberto Garciabad_offset = bad_sector * 512 314214faceSAlberto Garciablkdebug_file = os.path.join(iotests.test_dir, 'blkdebug.conf') 324214faceSAlberto Garcia 334214faceSAlberto Garciaclass BlockDeviceStatsTestCase(iotests.QMPTestCase): 34753b31b5SMax Reitz test_driver = "null-aio" 354214faceSAlberto Garcia total_rd_bytes = 0 364214faceSAlberto Garcia total_rd_ops = 0 374214faceSAlberto Garcia total_wr_bytes = 0 384214faceSAlberto Garcia total_wr_ops = 0 394214faceSAlberto Garcia total_wr_merged = 0 404214faceSAlberto Garcia total_flush_ops = 0 414214faceSAlberto Garcia failed_rd_ops = 0 424214faceSAlberto Garcia failed_wr_ops = 0 434214faceSAlberto Garcia invalid_rd_ops = 0 444214faceSAlberto Garcia invalid_wr_ops = 0 454214faceSAlberto Garcia wr_highest_offset = 0 464214faceSAlberto Garcia account_invalid = False 474214faceSAlberto Garcia account_failed = False 484214faceSAlberto Garcia 494214faceSAlberto Garcia def blockstats(self, device): 504214faceSAlberto Garcia result = self.vm.qmp("query-blockstats") 514214faceSAlberto Garcia for r in result['return']: 524214faceSAlberto Garcia if r['device'] == device: 534214faceSAlberto Garcia return r['stats'] 544214faceSAlberto Garcia raise Exception("Device not found for blockstats: %s" % device) 554214faceSAlberto Garcia 564214faceSAlberto Garcia def create_blkdebug_file(self): 574214faceSAlberto Garcia file = open(blkdebug_file, 'w') 584214faceSAlberto Garcia file.write(''' 594214faceSAlberto Garcia[inject-error] 604214faceSAlberto Garciaevent = "read_aio" 614214faceSAlberto Garciaerrno = "5" 624214faceSAlberto Garciasector = "%d" 634214faceSAlberto Garcia 644214faceSAlberto Garcia[inject-error] 654214faceSAlberto Garciaevent = "write_aio" 664214faceSAlberto Garciaerrno = "5" 674214faceSAlberto Garciasector = "%d" 684214faceSAlberto Garcia''' % (bad_sector, bad_sector)) 694214faceSAlberto Garcia file.close() 704214faceSAlberto Garcia 71753b31b5SMax Reitz def required_drivers(self): 72753b31b5SMax Reitz return [self.test_driver] 73753b31b5SMax Reitz 74753b31b5SMax Reitz @iotests.skip_if_unsupported(required_drivers) 754214faceSAlberto Garcia def setUp(self): 764214faceSAlberto Garcia drive_args = [] 7740119effSAlberto Garcia drive_args.append("stats-intervals.0=%d" % interval_length) 784214faceSAlberto Garcia drive_args.append("stats-account-invalid=%s" % 794214faceSAlberto Garcia (self.account_invalid and "on" or "off")) 804214faceSAlberto Garcia drive_args.append("stats-account-failed=%s" % 814214faceSAlberto Garcia (self.account_failed and "on" or "off")) 82a6862418SAndrey Shinkevich drive_args.append("file.image.read-zeroes=on") 834214faceSAlberto Garcia self.create_blkdebug_file() 84753b31b5SMax Reitz self.vm = iotests.VM().add_drive('blkdebug:%s:%s://' % 85753b31b5SMax Reitz (blkdebug_file, self.test_driver), 864214faceSAlberto Garcia ','.join(drive_args)) 874214faceSAlberto Garcia self.vm.launch() 884214faceSAlberto Garcia # Set an initial value for the clock 894214faceSAlberto Garcia self.vm.qtest("clock_step %d" % nsec_per_sec) 904214faceSAlberto Garcia 914214faceSAlberto Garcia def tearDown(self): 924214faceSAlberto Garcia self.vm.shutdown() 934214faceSAlberto Garcia os.remove(blkdebug_file) 944214faceSAlberto Garcia 954214faceSAlberto Garcia def accounted_ops(self, read = False, write = False, flush = False): 964214faceSAlberto Garcia ops = 0 974214faceSAlberto Garcia if write: 984214faceSAlberto Garcia ops += self.total_wr_ops 994214faceSAlberto Garcia if self.account_failed: 1004214faceSAlberto Garcia ops += self.failed_wr_ops 1014214faceSAlberto Garcia if self.account_invalid: 1024214faceSAlberto Garcia ops += self.invalid_wr_ops 1034214faceSAlberto Garcia if read: 1044214faceSAlberto Garcia ops += self.total_rd_ops 1054214faceSAlberto Garcia if self.account_failed: 1064214faceSAlberto Garcia ops += self.failed_rd_ops 1074214faceSAlberto Garcia if self.account_invalid: 1084214faceSAlberto Garcia ops += self.invalid_rd_ops 1094214faceSAlberto Garcia if flush: 1104214faceSAlberto Garcia ops += self.total_flush_ops 1114214faceSAlberto Garcia return ops 1124214faceSAlberto Garcia 1134214faceSAlberto Garcia def accounted_latency(self, read = False, write = False, flush = False): 1144214faceSAlberto Garcia latency = 0 1154214faceSAlberto Garcia if write: 1164214faceSAlberto Garcia latency += self.total_wr_ops * op_latency 1174214faceSAlberto Garcia if self.account_failed: 1184214faceSAlberto Garcia latency += self.failed_wr_ops * op_latency 1194214faceSAlberto Garcia if read: 1204214faceSAlberto Garcia latency += self.total_rd_ops * op_latency 1214214faceSAlberto Garcia if self.account_failed: 1224214faceSAlberto Garcia latency += self.failed_rd_ops * op_latency 1234214faceSAlberto Garcia if flush: 1244214faceSAlberto Garcia latency += self.total_flush_ops * op_latency 1254214faceSAlberto Garcia return latency 1264214faceSAlberto Garcia 1274214faceSAlberto Garcia def check_values(self): 1284214faceSAlberto Garcia stats = self.blockstats('drive0') 1294214faceSAlberto Garcia 1304214faceSAlberto Garcia # Check that the totals match with what we have calculated 1314214faceSAlberto Garcia self.assertEqual(self.total_rd_bytes, stats['rd_bytes']) 1324214faceSAlberto Garcia self.assertEqual(self.total_wr_bytes, stats['wr_bytes']) 1334214faceSAlberto Garcia self.assertEqual(self.total_rd_ops, stats['rd_operations']) 1344214faceSAlberto Garcia self.assertEqual(self.total_wr_ops, stats['wr_operations']) 1354214faceSAlberto Garcia self.assertEqual(self.total_flush_ops, stats['flush_operations']) 1364214faceSAlberto Garcia self.assertEqual(self.wr_highest_offset, stats['wr_highest_offset']) 1374214faceSAlberto Garcia self.assertEqual(self.failed_rd_ops, stats['failed_rd_operations']) 1384214faceSAlberto Garcia self.assertEqual(self.failed_wr_ops, stats['failed_wr_operations']) 1394214faceSAlberto Garcia self.assertEqual(self.invalid_rd_ops, stats['invalid_rd_operations']) 1404214faceSAlberto Garcia self.assertEqual(self.invalid_wr_ops, stats['invalid_wr_operations']) 1414214faceSAlberto Garcia self.assertEqual(self.account_invalid, stats['account_invalid']) 1424214faceSAlberto Garcia self.assertEqual(self.account_failed, stats['account_failed']) 1434214faceSAlberto Garcia self.assertEqual(self.total_wr_merged, stats['wr_merged']) 1444214faceSAlberto Garcia 1454214faceSAlberto Garcia # Check that there's exactly one interval with the length we defined 1464214faceSAlberto Garcia self.assertEqual(1, len(stats['timed_stats'])) 1474214faceSAlberto Garcia timed_stats = stats['timed_stats'][0] 1484214faceSAlberto Garcia self.assertEqual(interval_length, timed_stats['interval_length']) 1494214faceSAlberto Garcia 1504214faceSAlberto Garcia total_rd_latency = self.accounted_latency(read = True) 1514214faceSAlberto Garcia if (total_rd_latency != 0): 1524214faceSAlberto Garcia self.assertEqual(total_rd_latency, stats['rd_total_time_ns']) 1534214faceSAlberto Garcia self.assertEqual(op_latency, timed_stats['min_rd_latency_ns']) 1544214faceSAlberto Garcia self.assertEqual(op_latency, timed_stats['max_rd_latency_ns']) 1554214faceSAlberto Garcia self.assertEqual(op_latency, timed_stats['avg_rd_latency_ns']) 1564214faceSAlberto Garcia self.assertLess(0, timed_stats['avg_rd_queue_depth']) 1574214faceSAlberto Garcia else: 1584214faceSAlberto Garcia self.assertEqual(0, stats['rd_total_time_ns']) 1594214faceSAlberto Garcia self.assertEqual(0, timed_stats['min_rd_latency_ns']) 1604214faceSAlberto Garcia self.assertEqual(0, timed_stats['max_rd_latency_ns']) 1614214faceSAlberto Garcia self.assertEqual(0, timed_stats['avg_rd_latency_ns']) 1624214faceSAlberto Garcia self.assertEqual(0, timed_stats['avg_rd_queue_depth']) 1634214faceSAlberto Garcia 1644214faceSAlberto Garcia # min read latency <= avg read latency <= max read latency 1654214faceSAlberto Garcia self.assertLessEqual(timed_stats['min_rd_latency_ns'], 1664214faceSAlberto Garcia timed_stats['avg_rd_latency_ns']) 1674214faceSAlberto Garcia self.assertLessEqual(timed_stats['avg_rd_latency_ns'], 1684214faceSAlberto Garcia timed_stats['max_rd_latency_ns']) 1694214faceSAlberto Garcia 1704214faceSAlberto Garcia total_wr_latency = self.accounted_latency(write = True) 1714214faceSAlberto Garcia if (total_wr_latency != 0): 1724214faceSAlberto Garcia self.assertEqual(total_wr_latency, stats['wr_total_time_ns']) 1734214faceSAlberto Garcia self.assertEqual(op_latency, timed_stats['min_wr_latency_ns']) 1744214faceSAlberto Garcia self.assertEqual(op_latency, timed_stats['max_wr_latency_ns']) 1754214faceSAlberto Garcia self.assertEqual(op_latency, timed_stats['avg_wr_latency_ns']) 1764214faceSAlberto Garcia self.assertLess(0, timed_stats['avg_wr_queue_depth']) 1774214faceSAlberto Garcia else: 1784214faceSAlberto Garcia self.assertEqual(0, stats['wr_total_time_ns']) 1794214faceSAlberto Garcia self.assertEqual(0, timed_stats['min_wr_latency_ns']) 1804214faceSAlberto Garcia self.assertEqual(0, timed_stats['max_wr_latency_ns']) 1814214faceSAlberto Garcia self.assertEqual(0, timed_stats['avg_wr_latency_ns']) 1824214faceSAlberto Garcia self.assertEqual(0, timed_stats['avg_wr_queue_depth']) 1834214faceSAlberto Garcia 1844214faceSAlberto Garcia # min write latency <= avg write latency <= max write latency 1854214faceSAlberto Garcia self.assertLessEqual(timed_stats['min_wr_latency_ns'], 1864214faceSAlberto Garcia timed_stats['avg_wr_latency_ns']) 1874214faceSAlberto Garcia self.assertLessEqual(timed_stats['avg_wr_latency_ns'], 1884214faceSAlberto Garcia timed_stats['max_wr_latency_ns']) 1894214faceSAlberto Garcia 1904214faceSAlberto Garcia total_flush_latency = self.accounted_latency(flush = True) 1914214faceSAlberto Garcia if (total_flush_latency != 0): 1924214faceSAlberto Garcia self.assertEqual(total_flush_latency, stats['flush_total_time_ns']) 1934214faceSAlberto Garcia self.assertEqual(op_latency, timed_stats['min_flush_latency_ns']) 1944214faceSAlberto Garcia self.assertEqual(op_latency, timed_stats['max_flush_latency_ns']) 1954214faceSAlberto Garcia self.assertEqual(op_latency, timed_stats['avg_flush_latency_ns']) 1964214faceSAlberto Garcia else: 1974214faceSAlberto Garcia self.assertEqual(0, stats['flush_total_time_ns']) 1984214faceSAlberto Garcia self.assertEqual(0, timed_stats['min_flush_latency_ns']) 1994214faceSAlberto Garcia self.assertEqual(0, timed_stats['max_flush_latency_ns']) 2004214faceSAlberto Garcia self.assertEqual(0, timed_stats['avg_flush_latency_ns']) 2014214faceSAlberto Garcia 2024214faceSAlberto Garcia # min flush latency <= avg flush latency <= max flush latency 2034214faceSAlberto Garcia self.assertLessEqual(timed_stats['min_flush_latency_ns'], 2044214faceSAlberto Garcia timed_stats['avg_flush_latency_ns']) 2054214faceSAlberto Garcia self.assertLessEqual(timed_stats['avg_flush_latency_ns'], 2064214faceSAlberto Garcia timed_stats['max_flush_latency_ns']) 2074214faceSAlberto Garcia 2084214faceSAlberto Garcia # idle_time_ns must be > 0 if we have performed any operation 2094214faceSAlberto Garcia if (self.accounted_ops(read = True, write = True, flush = True) != 0): 2104214faceSAlberto Garcia self.assertLess(0, stats['idle_time_ns']) 2114214faceSAlberto Garcia else: 212d7a4228eSEduardo Habkost self.assertFalse('idle_time_ns' in stats) 2134214faceSAlberto Garcia 2144214faceSAlberto Garcia # This test does not alter these, so they must be all 0 2154214faceSAlberto Garcia self.assertEqual(0, stats['rd_merged']) 2164214faceSAlberto Garcia self.assertEqual(0, stats['failed_flush_operations']) 2174214faceSAlberto Garcia self.assertEqual(0, stats['invalid_flush_operations']) 2184214faceSAlberto Garcia 2194214faceSAlberto Garcia def do_test_stats(self, rd_size = 0, rd_ops = 0, wr_size = 0, wr_ops = 0, 2204214faceSAlberto Garcia flush_ops = 0, invalid_rd_ops = 0, invalid_wr_ops = 0, 2214214faceSAlberto Garcia failed_rd_ops = 0, failed_wr_ops = 0, wr_merged = 0): 2224214faceSAlberto Garcia # The 'ops' list will contain all the requested I/O operations 2234214faceSAlberto Garcia ops = [] 2244214faceSAlberto Garcia for i in range(rd_ops): 2254214faceSAlberto Garcia ops.append("aio_read %d %d" % (i * rd_size, rd_size)) 2264214faceSAlberto Garcia 2274214faceSAlberto Garcia for i in range(wr_ops): 2284214faceSAlberto Garcia ops.append("aio_write %d %d" % (i * wr_size, wr_size)) 2294214faceSAlberto Garcia 2304214faceSAlberto Garcia for i in range(flush_ops): 2314214faceSAlberto Garcia ops.append("aio_flush") 2324214faceSAlberto Garcia 2334214faceSAlberto Garcia highest_offset = wr_ops * wr_size 2344214faceSAlberto Garcia 23537546ff2SEric Blake for i in range(invalid_rd_ops): 23637546ff2SEric Blake ops.append("aio_read -i 0 512") 2374214faceSAlberto Garcia 23837546ff2SEric Blake for i in range(invalid_wr_ops): 23937546ff2SEric Blake ops.append("aio_write -i 0 512") 2404214faceSAlberto Garcia 2414214faceSAlberto Garcia for i in range(failed_rd_ops): 2424214faceSAlberto Garcia ops.append("aio_read %d 512" % bad_offset) 2434214faceSAlberto Garcia 2444214faceSAlberto Garcia for i in range(failed_wr_ops): 2454214faceSAlberto Garcia ops.append("aio_write %d 512" % bad_offset) 2464214faceSAlberto Garcia 24719026817SMax Reitz # We need an extra aio_flush to settle all outstanding AIO 24819026817SMax Reitz # operations before we can advance the virtual clock, so that 24919026817SMax Reitz # the last access happens before clock_step and idle_time_ns 25019026817SMax Reitz # will be greater than 0 25119026817SMax Reitz extra_flush = 0 25219026817SMax Reitz if rd_ops + wr_ops + invalid_rd_ops + invalid_wr_ops + \ 25319026817SMax Reitz failed_rd_ops + failed_wr_ops > 0: 25419026817SMax Reitz extra_flush = 1 25519026817SMax Reitz 25619026817SMax Reitz if extra_flush > 0: 25719026817SMax Reitz ops.append("aio_flush") 25819026817SMax Reitz 2594214faceSAlberto Garcia if failed_wr_ops > 0: 2604214faceSAlberto Garcia highest_offset = max(highest_offset, bad_offset + 512) 2614214faceSAlberto Garcia 2624214faceSAlberto Garcia # Now perform all operations 2634214faceSAlberto Garcia for op in ops: 2644214faceSAlberto Garcia self.vm.hmp_qemu_io("drive0", op) 2654214faceSAlberto Garcia 2664214faceSAlberto Garcia # Update the expected totals 2674214faceSAlberto Garcia self.total_rd_bytes += rd_ops * rd_size 2684214faceSAlberto Garcia self.total_rd_ops += rd_ops 2694214faceSAlberto Garcia self.total_wr_bytes += wr_ops * wr_size 2704214faceSAlberto Garcia self.total_wr_ops += wr_ops 2714214faceSAlberto Garcia self.total_wr_merged += wr_merged 27219026817SMax Reitz self.total_flush_ops += flush_ops + extra_flush 2734214faceSAlberto Garcia self.invalid_rd_ops += invalid_rd_ops 2744214faceSAlberto Garcia self.invalid_wr_ops += invalid_wr_ops 2754214faceSAlberto Garcia self.failed_rd_ops += failed_rd_ops 2764214faceSAlberto Garcia self.failed_wr_ops += failed_wr_ops 2774214faceSAlberto Garcia 2784214faceSAlberto Garcia self.wr_highest_offset = max(self.wr_highest_offset, highest_offset) 2794214faceSAlberto Garcia 2804214faceSAlberto Garcia # Advance the clock so idle_time_ns has a meaningful value 2814214faceSAlberto Garcia self.vm.qtest("clock_step %d" % nsec_per_sec) 2824214faceSAlberto Garcia 2834214faceSAlberto Garcia # And check that the actual statistics match the expected ones 2844214faceSAlberto Garcia self.check_values() 2854214faceSAlberto Garcia 2864214faceSAlberto Garcia def test_read_only(self): 2874214faceSAlberto Garcia test_values = [[512, 1], 2884214faceSAlberto Garcia [65536, 1], 2894214faceSAlberto Garcia [512, 12], 2904214faceSAlberto Garcia [65536, 12]] 2914214faceSAlberto Garcia for i in test_values: 2924214faceSAlberto Garcia self.do_test_stats(rd_size = i[0], rd_ops = i[1]) 2934214faceSAlberto Garcia 2944214faceSAlberto Garcia def test_write_only(self): 2954214faceSAlberto Garcia test_values = [[512, 1], 2964214faceSAlberto Garcia [65536, 1], 2974214faceSAlberto Garcia [512, 12], 2984214faceSAlberto Garcia [65536, 12]] 2994214faceSAlberto Garcia for i in test_values: 3004214faceSAlberto Garcia self.do_test_stats(wr_size = i[0], wr_ops = i[1]) 3014214faceSAlberto Garcia 3024214faceSAlberto Garcia def test_invalid(self): 3034214faceSAlberto Garcia self.do_test_stats(invalid_rd_ops = 7) 3044214faceSAlberto Garcia self.do_test_stats(invalid_wr_ops = 3) 3054214faceSAlberto Garcia self.do_test_stats(invalid_rd_ops = 4, invalid_wr_ops = 5) 3064214faceSAlberto Garcia 3074214faceSAlberto Garcia def test_failed(self): 3084214faceSAlberto Garcia self.do_test_stats(failed_rd_ops = 8) 3094214faceSAlberto Garcia self.do_test_stats(failed_wr_ops = 6) 3104214faceSAlberto Garcia self.do_test_stats(failed_rd_ops = 5, failed_wr_ops = 12) 3114214faceSAlberto Garcia 3124214faceSAlberto Garcia def test_flush(self): 3134214faceSAlberto Garcia self.do_test_stats(flush_ops = 8) 3144214faceSAlberto Garcia 3154214faceSAlberto Garcia def test_all(self): 3164214faceSAlberto Garcia # rd_size, rd_ops, wr_size, wr_ops, flush_ops 3174214faceSAlberto Garcia # invalid_rd_ops, invalid_wr_ops, 3184214faceSAlberto Garcia # failed_rd_ops, failed_wr_ops 3194214faceSAlberto Garcia # wr_merged 32091c6e4b7SKevin Wolf test_values = [[512, 1, 512, 1, 1, 4, 7, 5, 2, 0], 32191c6e4b7SKevin Wolf [65536, 1, 2048, 12, 7, 7, 5, 2, 5, 0], 32291c6e4b7SKevin Wolf [32768, 9, 8192, 1, 4, 3, 2, 4, 6, 0], 32391c6e4b7SKevin Wolf [16384, 11, 3584, 16, 9, 8, 6, 7, 3, 0]] 3244214faceSAlberto Garcia for i in test_values: 3254214faceSAlberto Garcia self.do_test_stats(*i) 3264214faceSAlberto Garcia 3274214faceSAlberto Garcia def test_no_op(self): 3284214faceSAlberto Garcia # All values must be sane before doing any I/O 3294214faceSAlberto Garcia self.check_values() 3304214faceSAlberto Garcia 3314214faceSAlberto Garcia 3324214faceSAlberto Garciaclass BlockDeviceStatsTestAccountInvalid(BlockDeviceStatsTestCase): 3334214faceSAlberto Garcia account_invalid = True 3344214faceSAlberto Garcia account_failed = False 3354214faceSAlberto Garcia 3364214faceSAlberto Garciaclass BlockDeviceStatsTestAccountFailed(BlockDeviceStatsTestCase): 3374214faceSAlberto Garcia account_invalid = False 3384214faceSAlberto Garcia account_failed = True 3394214faceSAlberto Garcia 3404214faceSAlberto Garciaclass BlockDeviceStatsTestAccountBoth(BlockDeviceStatsTestCase): 3414214faceSAlberto Garcia account_invalid = True 3424214faceSAlberto Garcia account_failed = True 3434214faceSAlberto Garcia 3444214faceSAlberto Garciaclass BlockDeviceStatsTestCoroutine(BlockDeviceStatsTestCase): 345753b31b5SMax Reitz test_driver = "null-co" 3464214faceSAlberto Garcia 3474214faceSAlberto Garciaif __name__ == '__main__': 348753b31b5SMax Reitz if 'null-co' not in iotests.supported_formats(): 349753b31b5SMax Reitz iotests.notrun('null-co driver support missing') 3504214faceSAlberto Garcia iotests.main(supported_fmts=["raw"]) 351