184092dbcSRoman Gushchin /* SPDX-License-Identifier: GPL-2.0 */ 284092dbcSRoman Gushchin #define _GNU_SOURCE 384092dbcSRoman Gushchin 484092dbcSRoman Gushchin #include <linux/limits.h> 584092dbcSRoman Gushchin #include <fcntl.h> 684092dbcSRoman Gushchin #include <stdio.h> 784092dbcSRoman Gushchin #include <stdlib.h> 884092dbcSRoman Gushchin #include <string.h> 984092dbcSRoman Gushchin #include <sys/stat.h> 1084092dbcSRoman Gushchin #include <sys/types.h> 1184092dbcSRoman Gushchin #include <unistd.h> 1284092dbcSRoman Gushchin 1384092dbcSRoman Gushchin #include "../kselftest.h" 1484092dbcSRoman Gushchin #include "cgroup_util.h" 1584092dbcSRoman Gushchin 1684092dbcSRoman Gushchin /* 1784092dbcSRoman Gushchin * This test creates two nested cgroups with and without enabling 1884092dbcSRoman Gushchin * the memory controller. 1984092dbcSRoman Gushchin */ 2084092dbcSRoman Gushchin static int test_memcg_subtree_control(const char *root) 2184092dbcSRoman Gushchin { 2284092dbcSRoman Gushchin char *parent, *child, *parent2, *child2; 2384092dbcSRoman Gushchin int ret = KSFT_FAIL; 2484092dbcSRoman Gushchin char buf[PAGE_SIZE]; 2584092dbcSRoman Gushchin 2684092dbcSRoman Gushchin /* Create two nested cgroups with the memory controller enabled */ 2784092dbcSRoman Gushchin parent = cg_name(root, "memcg_test_0"); 2884092dbcSRoman Gushchin child = cg_name(root, "memcg_test_0/memcg_test_1"); 2984092dbcSRoman Gushchin if (!parent || !child) 3084092dbcSRoman Gushchin goto cleanup; 3184092dbcSRoman Gushchin 3284092dbcSRoman Gushchin if (cg_create(parent)) 3384092dbcSRoman Gushchin goto cleanup; 3484092dbcSRoman Gushchin 3584092dbcSRoman Gushchin if (cg_write(parent, "cgroup.subtree_control", "+memory")) 3684092dbcSRoman Gushchin goto cleanup; 3784092dbcSRoman Gushchin 3884092dbcSRoman Gushchin if (cg_create(child)) 3984092dbcSRoman Gushchin goto cleanup; 4084092dbcSRoman Gushchin 4184092dbcSRoman Gushchin if (cg_read_strstr(child, "cgroup.controllers", "memory")) 4284092dbcSRoman Gushchin goto cleanup; 4384092dbcSRoman Gushchin 4484092dbcSRoman Gushchin /* Create two nested cgroups without enabling memory controller */ 4584092dbcSRoman Gushchin parent2 = cg_name(root, "memcg_test_1"); 4684092dbcSRoman Gushchin child2 = cg_name(root, "memcg_test_1/memcg_test_1"); 4784092dbcSRoman Gushchin if (!parent2 || !child2) 4884092dbcSRoman Gushchin goto cleanup; 4984092dbcSRoman Gushchin 5084092dbcSRoman Gushchin if (cg_create(parent2)) 5184092dbcSRoman Gushchin goto cleanup; 5284092dbcSRoman Gushchin 5384092dbcSRoman Gushchin if (cg_create(child2)) 5484092dbcSRoman Gushchin goto cleanup; 5584092dbcSRoman Gushchin 5684092dbcSRoman Gushchin if (cg_read(child2, "cgroup.controllers", buf, sizeof(buf))) 5784092dbcSRoman Gushchin goto cleanup; 5884092dbcSRoman Gushchin 5984092dbcSRoman Gushchin if (!cg_read_strstr(child2, "cgroup.controllers", "memory")) 6084092dbcSRoman Gushchin goto cleanup; 6184092dbcSRoman Gushchin 6284092dbcSRoman Gushchin ret = KSFT_PASS; 6384092dbcSRoman Gushchin 6484092dbcSRoman Gushchin cleanup: 6584092dbcSRoman Gushchin cg_destroy(child); 6684092dbcSRoman Gushchin cg_destroy(parent); 6784092dbcSRoman Gushchin free(parent); 6884092dbcSRoman Gushchin free(child); 6984092dbcSRoman Gushchin 7084092dbcSRoman Gushchin cg_destroy(child2); 7184092dbcSRoman Gushchin cg_destroy(parent2); 7284092dbcSRoman Gushchin free(parent2); 7384092dbcSRoman Gushchin free(child2); 7484092dbcSRoman Gushchin 7584092dbcSRoman Gushchin return ret; 7684092dbcSRoman Gushchin } 7784092dbcSRoman Gushchin 7884092dbcSRoman Gushchin static int alloc_anon_50M_check(const char *cgroup, void *arg) 7984092dbcSRoman Gushchin { 8084092dbcSRoman Gushchin size_t size = MB(50); 8184092dbcSRoman Gushchin char *buf, *ptr; 8284092dbcSRoman Gushchin long anon, current; 8384092dbcSRoman Gushchin int ret = -1; 8484092dbcSRoman Gushchin 8584092dbcSRoman Gushchin buf = malloc(size); 8684092dbcSRoman Gushchin for (ptr = buf; ptr < buf + size; ptr += PAGE_SIZE) 8784092dbcSRoman Gushchin *ptr = 0; 8884092dbcSRoman Gushchin 8984092dbcSRoman Gushchin current = cg_read_long(cgroup, "memory.current"); 9084092dbcSRoman Gushchin if (current < size) 9184092dbcSRoman Gushchin goto cleanup; 9284092dbcSRoman Gushchin 9384092dbcSRoman Gushchin if (!values_close(size, current, 3)) 9484092dbcSRoman Gushchin goto cleanup; 9584092dbcSRoman Gushchin 9684092dbcSRoman Gushchin anon = cg_read_key_long(cgroup, "memory.stat", "anon "); 9784092dbcSRoman Gushchin if (anon < 0) 9884092dbcSRoman Gushchin goto cleanup; 9984092dbcSRoman Gushchin 10084092dbcSRoman Gushchin if (!values_close(anon, current, 3)) 10184092dbcSRoman Gushchin goto cleanup; 10284092dbcSRoman Gushchin 10384092dbcSRoman Gushchin ret = 0; 10484092dbcSRoman Gushchin cleanup: 10584092dbcSRoman Gushchin free(buf); 10684092dbcSRoman Gushchin return ret; 10784092dbcSRoman Gushchin } 10884092dbcSRoman Gushchin 10984092dbcSRoman Gushchin static int alloc_pagecache_50M_check(const char *cgroup, void *arg) 11084092dbcSRoman Gushchin { 11184092dbcSRoman Gushchin size_t size = MB(50); 11284092dbcSRoman Gushchin int ret = -1; 11384092dbcSRoman Gushchin long current, file; 11484092dbcSRoman Gushchin int fd; 11584092dbcSRoman Gushchin 11684092dbcSRoman Gushchin fd = get_temp_fd(); 11784092dbcSRoman Gushchin if (fd < 0) 11884092dbcSRoman Gushchin return -1; 11984092dbcSRoman Gushchin 12084092dbcSRoman Gushchin if (alloc_pagecache(fd, size)) 12184092dbcSRoman Gushchin goto cleanup; 12284092dbcSRoman Gushchin 12384092dbcSRoman Gushchin current = cg_read_long(cgroup, "memory.current"); 12484092dbcSRoman Gushchin if (current < size) 12584092dbcSRoman Gushchin goto cleanup; 12684092dbcSRoman Gushchin 12784092dbcSRoman Gushchin file = cg_read_key_long(cgroup, "memory.stat", "file "); 12884092dbcSRoman Gushchin if (file < 0) 12984092dbcSRoman Gushchin goto cleanup; 13084092dbcSRoman Gushchin 13184092dbcSRoman Gushchin if (!values_close(file, current, 10)) 13284092dbcSRoman Gushchin goto cleanup; 13384092dbcSRoman Gushchin 13484092dbcSRoman Gushchin ret = 0; 13584092dbcSRoman Gushchin 13684092dbcSRoman Gushchin cleanup: 13784092dbcSRoman Gushchin close(fd); 13884092dbcSRoman Gushchin return ret; 13984092dbcSRoman Gushchin } 14084092dbcSRoman Gushchin 14184092dbcSRoman Gushchin /* 14284092dbcSRoman Gushchin * This test create a memory cgroup, allocates 14384092dbcSRoman Gushchin * some anonymous memory and some pagecache 14484092dbcSRoman Gushchin * and check memory.current and some memory.stat values. 14584092dbcSRoman Gushchin */ 14684092dbcSRoman Gushchin static int test_memcg_current(const char *root) 14784092dbcSRoman Gushchin { 14884092dbcSRoman Gushchin int ret = KSFT_FAIL; 14984092dbcSRoman Gushchin long current; 15084092dbcSRoman Gushchin char *memcg; 15184092dbcSRoman Gushchin 15284092dbcSRoman Gushchin memcg = cg_name(root, "memcg_test"); 15384092dbcSRoman Gushchin if (!memcg) 15484092dbcSRoman Gushchin goto cleanup; 15584092dbcSRoman Gushchin 15684092dbcSRoman Gushchin if (cg_create(memcg)) 15784092dbcSRoman Gushchin goto cleanup; 15884092dbcSRoman Gushchin 15984092dbcSRoman Gushchin current = cg_read_long(memcg, "memory.current"); 16084092dbcSRoman Gushchin if (current != 0) 16184092dbcSRoman Gushchin goto cleanup; 16284092dbcSRoman Gushchin 16384092dbcSRoman Gushchin if (cg_run(memcg, alloc_anon_50M_check, NULL)) 16484092dbcSRoman Gushchin goto cleanup; 16584092dbcSRoman Gushchin 16684092dbcSRoman Gushchin if (cg_run(memcg, alloc_pagecache_50M_check, NULL)) 16784092dbcSRoman Gushchin goto cleanup; 16884092dbcSRoman Gushchin 16984092dbcSRoman Gushchin ret = KSFT_PASS; 17084092dbcSRoman Gushchin 17184092dbcSRoman Gushchin cleanup: 17284092dbcSRoman Gushchin cg_destroy(memcg); 17384092dbcSRoman Gushchin free(memcg); 17484092dbcSRoman Gushchin 17584092dbcSRoman Gushchin return ret; 17684092dbcSRoman Gushchin } 17784092dbcSRoman Gushchin 17884092dbcSRoman Gushchin static int alloc_pagecache_50M(const char *cgroup, void *arg) 17984092dbcSRoman Gushchin { 18084092dbcSRoman Gushchin int fd = (long)arg; 18184092dbcSRoman Gushchin 18284092dbcSRoman Gushchin return alloc_pagecache(fd, MB(50)); 18384092dbcSRoman Gushchin } 18484092dbcSRoman Gushchin 18584092dbcSRoman Gushchin static int alloc_pagecache_50M_noexit(const char *cgroup, void *arg) 18684092dbcSRoman Gushchin { 18784092dbcSRoman Gushchin int fd = (long)arg; 18884092dbcSRoman Gushchin int ppid = getppid(); 18984092dbcSRoman Gushchin 19084092dbcSRoman Gushchin if (alloc_pagecache(fd, MB(50))) 19184092dbcSRoman Gushchin return -1; 19284092dbcSRoman Gushchin 19384092dbcSRoman Gushchin while (getppid() == ppid) 19484092dbcSRoman Gushchin sleep(1); 19584092dbcSRoman Gushchin 19684092dbcSRoman Gushchin return 0; 19784092dbcSRoman Gushchin } 19884092dbcSRoman Gushchin 19984092dbcSRoman Gushchin /* 20084092dbcSRoman Gushchin * First, this test creates the following hierarchy: 20184092dbcSRoman Gushchin * A memory.min = 50M, memory.max = 200M 20284092dbcSRoman Gushchin * A/B memory.min = 50M, memory.current = 50M 20384092dbcSRoman Gushchin * A/B/C memory.min = 75M, memory.current = 50M 20484092dbcSRoman Gushchin * A/B/D memory.min = 25M, memory.current = 50M 20584092dbcSRoman Gushchin * A/B/E memory.min = 500M, memory.current = 0 20684092dbcSRoman Gushchin * A/B/F memory.min = 0, memory.current = 50M 20784092dbcSRoman Gushchin * 20884092dbcSRoman Gushchin * Usages are pagecache, but the test keeps a running 20984092dbcSRoman Gushchin * process in every leaf cgroup. 21084092dbcSRoman Gushchin * Then it creates A/G and creates a significant 21184092dbcSRoman Gushchin * memory pressure in it. 21284092dbcSRoman Gushchin * 21384092dbcSRoman Gushchin * A/B memory.current ~= 50M 21484092dbcSRoman Gushchin * A/B/C memory.current ~= 33M 21584092dbcSRoman Gushchin * A/B/D memory.current ~= 17M 21684092dbcSRoman Gushchin * A/B/E memory.current ~= 0 21784092dbcSRoman Gushchin * 21884092dbcSRoman Gushchin * After that it tries to allocate more than there is 21984092dbcSRoman Gushchin * unprotected memory in A available, and checks 22084092dbcSRoman Gushchin * checks that memory.min protects pagecache even 22184092dbcSRoman Gushchin * in this case. 22284092dbcSRoman Gushchin */ 22384092dbcSRoman Gushchin static int test_memcg_min(const char *root) 22484092dbcSRoman Gushchin { 22584092dbcSRoman Gushchin int ret = KSFT_FAIL; 22684092dbcSRoman Gushchin char *parent[3] = {NULL}; 22784092dbcSRoman Gushchin char *children[4] = {NULL}; 22884092dbcSRoman Gushchin long c[4]; 22984092dbcSRoman Gushchin int i, attempts; 23084092dbcSRoman Gushchin int fd; 23184092dbcSRoman Gushchin 23284092dbcSRoman Gushchin fd = get_temp_fd(); 23384092dbcSRoman Gushchin if (fd < 0) 23484092dbcSRoman Gushchin goto cleanup; 23584092dbcSRoman Gushchin 23684092dbcSRoman Gushchin parent[0] = cg_name(root, "memcg_test_0"); 23784092dbcSRoman Gushchin if (!parent[0]) 23884092dbcSRoman Gushchin goto cleanup; 23984092dbcSRoman Gushchin 24084092dbcSRoman Gushchin parent[1] = cg_name(parent[0], "memcg_test_1"); 24184092dbcSRoman Gushchin if (!parent[1]) 24284092dbcSRoman Gushchin goto cleanup; 24384092dbcSRoman Gushchin 24484092dbcSRoman Gushchin parent[2] = cg_name(parent[0], "memcg_test_2"); 24584092dbcSRoman Gushchin if (!parent[2]) 24684092dbcSRoman Gushchin goto cleanup; 24784092dbcSRoman Gushchin 24884092dbcSRoman Gushchin if (cg_create(parent[0])) 24984092dbcSRoman Gushchin goto cleanup; 25084092dbcSRoman Gushchin 25184092dbcSRoman Gushchin if (cg_read_long(parent[0], "memory.min")) { 25284092dbcSRoman Gushchin ret = KSFT_SKIP; 25384092dbcSRoman Gushchin goto cleanup; 25484092dbcSRoman Gushchin } 25584092dbcSRoman Gushchin 25684092dbcSRoman Gushchin if (cg_write(parent[0], "cgroup.subtree_control", "+memory")) 25784092dbcSRoman Gushchin goto cleanup; 25884092dbcSRoman Gushchin 25984092dbcSRoman Gushchin if (cg_write(parent[0], "memory.max", "200M")) 26084092dbcSRoman Gushchin goto cleanup; 26184092dbcSRoman Gushchin 26284092dbcSRoman Gushchin if (cg_write(parent[0], "memory.swap.max", "0")) 26384092dbcSRoman Gushchin goto cleanup; 26484092dbcSRoman Gushchin 26584092dbcSRoman Gushchin if (cg_create(parent[1])) 26684092dbcSRoman Gushchin goto cleanup; 26784092dbcSRoman Gushchin 26884092dbcSRoman Gushchin if (cg_write(parent[1], "cgroup.subtree_control", "+memory")) 26984092dbcSRoman Gushchin goto cleanup; 27084092dbcSRoman Gushchin 27184092dbcSRoman Gushchin if (cg_create(parent[2])) 27284092dbcSRoman Gushchin goto cleanup; 27384092dbcSRoman Gushchin 27484092dbcSRoman Gushchin for (i = 0; i < ARRAY_SIZE(children); i++) { 27584092dbcSRoman Gushchin children[i] = cg_name_indexed(parent[1], "child_memcg", i); 27684092dbcSRoman Gushchin if (!children[i]) 27784092dbcSRoman Gushchin goto cleanup; 27884092dbcSRoman Gushchin 27984092dbcSRoman Gushchin if (cg_create(children[i])) 28084092dbcSRoman Gushchin goto cleanup; 28184092dbcSRoman Gushchin 28284092dbcSRoman Gushchin if (i == 2) 28384092dbcSRoman Gushchin continue; 28484092dbcSRoman Gushchin 28584092dbcSRoman Gushchin cg_run_nowait(children[i], alloc_pagecache_50M_noexit, 28684092dbcSRoman Gushchin (void *)(long)fd); 28784092dbcSRoman Gushchin } 28884092dbcSRoman Gushchin 28984092dbcSRoman Gushchin if (cg_write(parent[0], "memory.min", "50M")) 29084092dbcSRoman Gushchin goto cleanup; 29184092dbcSRoman Gushchin if (cg_write(parent[1], "memory.min", "50M")) 29284092dbcSRoman Gushchin goto cleanup; 29384092dbcSRoman Gushchin if (cg_write(children[0], "memory.min", "75M")) 29484092dbcSRoman Gushchin goto cleanup; 29584092dbcSRoman Gushchin if (cg_write(children[1], "memory.min", "25M")) 29684092dbcSRoman Gushchin goto cleanup; 29784092dbcSRoman Gushchin if (cg_write(children[2], "memory.min", "500M")) 29884092dbcSRoman Gushchin goto cleanup; 29984092dbcSRoman Gushchin if (cg_write(children[3], "memory.min", "0")) 30084092dbcSRoman Gushchin goto cleanup; 30184092dbcSRoman Gushchin 30284092dbcSRoman Gushchin attempts = 0; 30384092dbcSRoman Gushchin while (!values_close(cg_read_long(parent[1], "memory.current"), 30484092dbcSRoman Gushchin MB(150), 3)) { 30584092dbcSRoman Gushchin if (attempts++ > 5) 30684092dbcSRoman Gushchin break; 30784092dbcSRoman Gushchin sleep(1); 30884092dbcSRoman Gushchin } 30984092dbcSRoman Gushchin 31084092dbcSRoman Gushchin if (cg_run(parent[2], alloc_anon, (void *)MB(148))) 31184092dbcSRoman Gushchin goto cleanup; 31284092dbcSRoman Gushchin 31384092dbcSRoman Gushchin if (!values_close(cg_read_long(parent[1], "memory.current"), MB(50), 3)) 31484092dbcSRoman Gushchin goto cleanup; 31584092dbcSRoman Gushchin 31684092dbcSRoman Gushchin for (i = 0; i < ARRAY_SIZE(children); i++) 31784092dbcSRoman Gushchin c[i] = cg_read_long(children[i], "memory.current"); 31884092dbcSRoman Gushchin 31984092dbcSRoman Gushchin if (!values_close(c[0], MB(33), 10)) 32084092dbcSRoman Gushchin goto cleanup; 32184092dbcSRoman Gushchin 32284092dbcSRoman Gushchin if (!values_close(c[1], MB(17), 10)) 32384092dbcSRoman Gushchin goto cleanup; 32484092dbcSRoman Gushchin 32584092dbcSRoman Gushchin if (!values_close(c[2], 0, 1)) 32684092dbcSRoman Gushchin goto cleanup; 32784092dbcSRoman Gushchin 32884092dbcSRoman Gushchin if (!cg_run(parent[2], alloc_anon, (void *)MB(170))) 32984092dbcSRoman Gushchin goto cleanup; 33084092dbcSRoman Gushchin 33184092dbcSRoman Gushchin if (!values_close(cg_read_long(parent[1], "memory.current"), MB(50), 3)) 33284092dbcSRoman Gushchin goto cleanup; 33384092dbcSRoman Gushchin 33484092dbcSRoman Gushchin ret = KSFT_PASS; 33584092dbcSRoman Gushchin 33684092dbcSRoman Gushchin cleanup: 33784092dbcSRoman Gushchin for (i = ARRAY_SIZE(children) - 1; i >= 0; i--) { 33884092dbcSRoman Gushchin if (!children[i]) 33984092dbcSRoman Gushchin continue; 34084092dbcSRoman Gushchin 34184092dbcSRoman Gushchin cg_destroy(children[i]); 34284092dbcSRoman Gushchin free(children[i]); 34384092dbcSRoman Gushchin } 34484092dbcSRoman Gushchin 34584092dbcSRoman Gushchin for (i = ARRAY_SIZE(parent) - 1; i >= 0; i--) { 34684092dbcSRoman Gushchin if (!parent[i]) 34784092dbcSRoman Gushchin continue; 34884092dbcSRoman Gushchin 34984092dbcSRoman Gushchin cg_destroy(parent[i]); 35084092dbcSRoman Gushchin free(parent[i]); 35184092dbcSRoman Gushchin } 35284092dbcSRoman Gushchin close(fd); 35384092dbcSRoman Gushchin return ret; 35484092dbcSRoman Gushchin } 35584092dbcSRoman Gushchin 35684092dbcSRoman Gushchin /* 35784092dbcSRoman Gushchin * First, this test creates the following hierarchy: 35884092dbcSRoman Gushchin * A memory.low = 50M, memory.max = 200M 35984092dbcSRoman Gushchin * A/B memory.low = 50M, memory.current = 50M 36084092dbcSRoman Gushchin * A/B/C memory.low = 75M, memory.current = 50M 36184092dbcSRoman Gushchin * A/B/D memory.low = 25M, memory.current = 50M 36284092dbcSRoman Gushchin * A/B/E memory.low = 500M, memory.current = 0 36384092dbcSRoman Gushchin * A/B/F memory.low = 0, memory.current = 50M 36484092dbcSRoman Gushchin * 36584092dbcSRoman Gushchin * Usages are pagecache. 36684092dbcSRoman Gushchin * Then it creates A/G an creates a significant 36784092dbcSRoman Gushchin * memory pressure in it. 36884092dbcSRoman Gushchin * 36984092dbcSRoman Gushchin * Then it checks actual memory usages and expects that: 37084092dbcSRoman Gushchin * A/B memory.current ~= 50M 37184092dbcSRoman Gushchin * A/B/ memory.current ~= 33M 37284092dbcSRoman Gushchin * A/B/D memory.current ~= 17M 37384092dbcSRoman Gushchin * A/B/E memory.current ~= 0 37484092dbcSRoman Gushchin * 37584092dbcSRoman Gushchin * After that it tries to allocate more than there is 37684092dbcSRoman Gushchin * unprotected memory in A available, 37784092dbcSRoman Gushchin * and checks low and oom events in memory.events. 37884092dbcSRoman Gushchin */ 37984092dbcSRoman Gushchin static int test_memcg_low(const char *root) 38084092dbcSRoman Gushchin { 38184092dbcSRoman Gushchin int ret = KSFT_FAIL; 38284092dbcSRoman Gushchin char *parent[3] = {NULL}; 38384092dbcSRoman Gushchin char *children[4] = {NULL}; 38484092dbcSRoman Gushchin long low, oom; 38584092dbcSRoman Gushchin long c[4]; 38684092dbcSRoman Gushchin int i; 38784092dbcSRoman Gushchin int fd; 38884092dbcSRoman Gushchin 38984092dbcSRoman Gushchin fd = get_temp_fd(); 39084092dbcSRoman Gushchin if (fd < 0) 39184092dbcSRoman Gushchin goto cleanup; 39284092dbcSRoman Gushchin 39384092dbcSRoman Gushchin parent[0] = cg_name(root, "memcg_test_0"); 39484092dbcSRoman Gushchin if (!parent[0]) 39584092dbcSRoman Gushchin goto cleanup; 39684092dbcSRoman Gushchin 39784092dbcSRoman Gushchin parent[1] = cg_name(parent[0], "memcg_test_1"); 39884092dbcSRoman Gushchin if (!parent[1]) 39984092dbcSRoman Gushchin goto cleanup; 40084092dbcSRoman Gushchin 40184092dbcSRoman Gushchin parent[2] = cg_name(parent[0], "memcg_test_2"); 40284092dbcSRoman Gushchin if (!parent[2]) 40384092dbcSRoman Gushchin goto cleanup; 40484092dbcSRoman Gushchin 40584092dbcSRoman Gushchin if (cg_create(parent[0])) 40684092dbcSRoman Gushchin goto cleanup; 40784092dbcSRoman Gushchin 40884092dbcSRoman Gushchin if (cg_read_long(parent[0], "memory.low")) 40984092dbcSRoman Gushchin goto cleanup; 41084092dbcSRoman Gushchin 41184092dbcSRoman Gushchin if (cg_write(parent[0], "cgroup.subtree_control", "+memory")) 41284092dbcSRoman Gushchin goto cleanup; 41384092dbcSRoman Gushchin 41484092dbcSRoman Gushchin if (cg_write(parent[0], "memory.max", "200M")) 41584092dbcSRoman Gushchin goto cleanup; 41684092dbcSRoman Gushchin 41784092dbcSRoman Gushchin if (cg_write(parent[0], "memory.swap.max", "0")) 41884092dbcSRoman Gushchin goto cleanup; 41984092dbcSRoman Gushchin 42084092dbcSRoman Gushchin if (cg_create(parent[1])) 42184092dbcSRoman Gushchin goto cleanup; 42284092dbcSRoman Gushchin 42384092dbcSRoman Gushchin if (cg_write(parent[1], "cgroup.subtree_control", "+memory")) 42484092dbcSRoman Gushchin goto cleanup; 42584092dbcSRoman Gushchin 42684092dbcSRoman Gushchin if (cg_create(parent[2])) 42784092dbcSRoman Gushchin goto cleanup; 42884092dbcSRoman Gushchin 42984092dbcSRoman Gushchin for (i = 0; i < ARRAY_SIZE(children); i++) { 43084092dbcSRoman Gushchin children[i] = cg_name_indexed(parent[1], "child_memcg", i); 43184092dbcSRoman Gushchin if (!children[i]) 43284092dbcSRoman Gushchin goto cleanup; 43384092dbcSRoman Gushchin 43484092dbcSRoman Gushchin if (cg_create(children[i])) 43584092dbcSRoman Gushchin goto cleanup; 43684092dbcSRoman Gushchin 43784092dbcSRoman Gushchin if (i == 2) 43884092dbcSRoman Gushchin continue; 43984092dbcSRoman Gushchin 44084092dbcSRoman Gushchin if (cg_run(children[i], alloc_pagecache_50M, (void *)(long)fd)) 44184092dbcSRoman Gushchin goto cleanup; 44284092dbcSRoman Gushchin } 44384092dbcSRoman Gushchin 44484092dbcSRoman Gushchin if (cg_write(parent[0], "memory.low", "50M")) 44584092dbcSRoman Gushchin goto cleanup; 44684092dbcSRoman Gushchin if (cg_write(parent[1], "memory.low", "50M")) 44784092dbcSRoman Gushchin goto cleanup; 44884092dbcSRoman Gushchin if (cg_write(children[0], "memory.low", "75M")) 44984092dbcSRoman Gushchin goto cleanup; 45084092dbcSRoman Gushchin if (cg_write(children[1], "memory.low", "25M")) 45184092dbcSRoman Gushchin goto cleanup; 45284092dbcSRoman Gushchin if (cg_write(children[2], "memory.low", "500M")) 45384092dbcSRoman Gushchin goto cleanup; 45484092dbcSRoman Gushchin if (cg_write(children[3], "memory.low", "0")) 45584092dbcSRoman Gushchin goto cleanup; 45684092dbcSRoman Gushchin 45784092dbcSRoman Gushchin if (cg_run(parent[2], alloc_anon, (void *)MB(148))) 45884092dbcSRoman Gushchin goto cleanup; 45984092dbcSRoman Gushchin 46084092dbcSRoman Gushchin if (!values_close(cg_read_long(parent[1], "memory.current"), MB(50), 3)) 46184092dbcSRoman Gushchin goto cleanup; 46284092dbcSRoman Gushchin 46384092dbcSRoman Gushchin for (i = 0; i < ARRAY_SIZE(children); i++) 46484092dbcSRoman Gushchin c[i] = cg_read_long(children[i], "memory.current"); 46584092dbcSRoman Gushchin 46684092dbcSRoman Gushchin if (!values_close(c[0], MB(33), 10)) 46784092dbcSRoman Gushchin goto cleanup; 46884092dbcSRoman Gushchin 46984092dbcSRoman Gushchin if (!values_close(c[1], MB(17), 10)) 47084092dbcSRoman Gushchin goto cleanup; 47184092dbcSRoman Gushchin 47284092dbcSRoman Gushchin if (!values_close(c[2], 0, 1)) 47384092dbcSRoman Gushchin goto cleanup; 47484092dbcSRoman Gushchin 47584092dbcSRoman Gushchin if (cg_run(parent[2], alloc_anon, (void *)MB(166))) { 47684092dbcSRoman Gushchin fprintf(stderr, 47784092dbcSRoman Gushchin "memory.low prevents from allocating anon memory\n"); 47884092dbcSRoman Gushchin goto cleanup; 47984092dbcSRoman Gushchin } 48084092dbcSRoman Gushchin 48184092dbcSRoman Gushchin for (i = 0; i < ARRAY_SIZE(children); i++) { 48284092dbcSRoman Gushchin oom = cg_read_key_long(children[i], "memory.events", "oom "); 48384092dbcSRoman Gushchin low = cg_read_key_long(children[i], "memory.events", "low "); 48484092dbcSRoman Gushchin 48584092dbcSRoman Gushchin if (oom) 48684092dbcSRoman Gushchin goto cleanup; 48784092dbcSRoman Gushchin if (i < 2 && low <= 0) 48884092dbcSRoman Gushchin goto cleanup; 48984092dbcSRoman Gushchin if (i >= 2 && low) 49084092dbcSRoman Gushchin goto cleanup; 49184092dbcSRoman Gushchin } 49284092dbcSRoman Gushchin 49384092dbcSRoman Gushchin ret = KSFT_PASS; 49484092dbcSRoman Gushchin 49584092dbcSRoman Gushchin cleanup: 49684092dbcSRoman Gushchin for (i = ARRAY_SIZE(children) - 1; i >= 0; i--) { 49784092dbcSRoman Gushchin if (!children[i]) 49884092dbcSRoman Gushchin continue; 49984092dbcSRoman Gushchin 50084092dbcSRoman Gushchin cg_destroy(children[i]); 50184092dbcSRoman Gushchin free(children[i]); 50284092dbcSRoman Gushchin } 50384092dbcSRoman Gushchin 50484092dbcSRoman Gushchin for (i = ARRAY_SIZE(parent) - 1; i >= 0; i--) { 50584092dbcSRoman Gushchin if (!parent[i]) 50684092dbcSRoman Gushchin continue; 50784092dbcSRoman Gushchin 50884092dbcSRoman Gushchin cg_destroy(parent[i]); 50984092dbcSRoman Gushchin free(parent[i]); 51084092dbcSRoman Gushchin } 51184092dbcSRoman Gushchin close(fd); 51284092dbcSRoman Gushchin return ret; 51384092dbcSRoman Gushchin } 51484092dbcSRoman Gushchin 51584092dbcSRoman Gushchin static int alloc_pagecache_max_30M(const char *cgroup, void *arg) 51684092dbcSRoman Gushchin { 51784092dbcSRoman Gushchin size_t size = MB(50); 51884092dbcSRoman Gushchin int ret = -1; 51984092dbcSRoman Gushchin long current; 52084092dbcSRoman Gushchin int fd; 52184092dbcSRoman Gushchin 52284092dbcSRoman Gushchin fd = get_temp_fd(); 52384092dbcSRoman Gushchin if (fd < 0) 52484092dbcSRoman Gushchin return -1; 52584092dbcSRoman Gushchin 52684092dbcSRoman Gushchin if (alloc_pagecache(fd, size)) 52784092dbcSRoman Gushchin goto cleanup; 52884092dbcSRoman Gushchin 52984092dbcSRoman Gushchin current = cg_read_long(cgroup, "memory.current"); 53084092dbcSRoman Gushchin if (current <= MB(29) || current > MB(30)) 53184092dbcSRoman Gushchin goto cleanup; 53284092dbcSRoman Gushchin 53384092dbcSRoman Gushchin ret = 0; 53484092dbcSRoman Gushchin 53584092dbcSRoman Gushchin cleanup: 53684092dbcSRoman Gushchin close(fd); 53784092dbcSRoman Gushchin return ret; 53884092dbcSRoman Gushchin 53984092dbcSRoman Gushchin } 54084092dbcSRoman Gushchin 54184092dbcSRoman Gushchin /* 54284092dbcSRoman Gushchin * This test checks that memory.high limits the amount of 54384092dbcSRoman Gushchin * memory which can be consumed by either anonymous memory 54484092dbcSRoman Gushchin * or pagecache. 54584092dbcSRoman Gushchin */ 54684092dbcSRoman Gushchin static int test_memcg_high(const char *root) 54784092dbcSRoman Gushchin { 54884092dbcSRoman Gushchin int ret = KSFT_FAIL; 54984092dbcSRoman Gushchin char *memcg; 55084092dbcSRoman Gushchin long high; 55184092dbcSRoman Gushchin 55284092dbcSRoman Gushchin memcg = cg_name(root, "memcg_test"); 55384092dbcSRoman Gushchin if (!memcg) 55484092dbcSRoman Gushchin goto cleanup; 55584092dbcSRoman Gushchin 55684092dbcSRoman Gushchin if (cg_create(memcg)) 55784092dbcSRoman Gushchin goto cleanup; 55884092dbcSRoman Gushchin 55984092dbcSRoman Gushchin if (cg_read_strcmp(memcg, "memory.high", "max\n")) 56084092dbcSRoman Gushchin goto cleanup; 56184092dbcSRoman Gushchin 56284092dbcSRoman Gushchin if (cg_write(memcg, "memory.swap.max", "0")) 56384092dbcSRoman Gushchin goto cleanup; 56484092dbcSRoman Gushchin 56584092dbcSRoman Gushchin if (cg_write(memcg, "memory.high", "30M")) 56684092dbcSRoman Gushchin goto cleanup; 56784092dbcSRoman Gushchin 56884092dbcSRoman Gushchin if (cg_run(memcg, alloc_anon, (void *)MB(100))) 56984092dbcSRoman Gushchin goto cleanup; 57084092dbcSRoman Gushchin 57184092dbcSRoman Gushchin if (!cg_run(memcg, alloc_pagecache_50M_check, NULL)) 57284092dbcSRoman Gushchin goto cleanup; 57384092dbcSRoman Gushchin 57484092dbcSRoman Gushchin if (cg_run(memcg, alloc_pagecache_max_30M, NULL)) 57584092dbcSRoman Gushchin goto cleanup; 57684092dbcSRoman Gushchin 57784092dbcSRoman Gushchin high = cg_read_key_long(memcg, "memory.events", "high "); 57884092dbcSRoman Gushchin if (high <= 0) 57984092dbcSRoman Gushchin goto cleanup; 58084092dbcSRoman Gushchin 58184092dbcSRoman Gushchin ret = KSFT_PASS; 58284092dbcSRoman Gushchin 58384092dbcSRoman Gushchin cleanup: 58484092dbcSRoman Gushchin cg_destroy(memcg); 58584092dbcSRoman Gushchin free(memcg); 58684092dbcSRoman Gushchin 58784092dbcSRoman Gushchin return ret; 58884092dbcSRoman Gushchin } 58984092dbcSRoman Gushchin 59084092dbcSRoman Gushchin /* 59184092dbcSRoman Gushchin * This test checks that memory.max limits the amount of 59284092dbcSRoman Gushchin * memory which can be consumed by either anonymous memory 59384092dbcSRoman Gushchin * or pagecache. 59484092dbcSRoman Gushchin */ 59584092dbcSRoman Gushchin static int test_memcg_max(const char *root) 59684092dbcSRoman Gushchin { 59784092dbcSRoman Gushchin int ret = KSFT_FAIL; 59884092dbcSRoman Gushchin char *memcg; 59984092dbcSRoman Gushchin long current, max; 60084092dbcSRoman Gushchin 60184092dbcSRoman Gushchin memcg = cg_name(root, "memcg_test"); 60284092dbcSRoman Gushchin if (!memcg) 60384092dbcSRoman Gushchin goto cleanup; 60484092dbcSRoman Gushchin 60584092dbcSRoman Gushchin if (cg_create(memcg)) 60684092dbcSRoman Gushchin goto cleanup; 60784092dbcSRoman Gushchin 60884092dbcSRoman Gushchin if (cg_read_strcmp(memcg, "memory.max", "max\n")) 60984092dbcSRoman Gushchin goto cleanup; 61084092dbcSRoman Gushchin 61184092dbcSRoman Gushchin if (cg_write(memcg, "memory.swap.max", "0")) 61284092dbcSRoman Gushchin goto cleanup; 61384092dbcSRoman Gushchin 61484092dbcSRoman Gushchin if (cg_write(memcg, "memory.max", "30M")) 61584092dbcSRoman Gushchin goto cleanup; 61684092dbcSRoman Gushchin 61784092dbcSRoman Gushchin /* Should be killed by OOM killer */ 61884092dbcSRoman Gushchin if (!cg_run(memcg, alloc_anon, (void *)MB(100))) 61984092dbcSRoman Gushchin goto cleanup; 62084092dbcSRoman Gushchin 62184092dbcSRoman Gushchin if (cg_run(memcg, alloc_pagecache_max_30M, NULL)) 62284092dbcSRoman Gushchin goto cleanup; 62384092dbcSRoman Gushchin 62484092dbcSRoman Gushchin current = cg_read_long(memcg, "memory.current"); 62584092dbcSRoman Gushchin if (current > MB(30) || !current) 62684092dbcSRoman Gushchin goto cleanup; 62784092dbcSRoman Gushchin 62884092dbcSRoman Gushchin max = cg_read_key_long(memcg, "memory.events", "max "); 62984092dbcSRoman Gushchin if (max <= 0) 63084092dbcSRoman Gushchin goto cleanup; 63184092dbcSRoman Gushchin 63284092dbcSRoman Gushchin ret = KSFT_PASS; 63384092dbcSRoman Gushchin 63484092dbcSRoman Gushchin cleanup: 63584092dbcSRoman Gushchin cg_destroy(memcg); 63684092dbcSRoman Gushchin free(memcg); 63784092dbcSRoman Gushchin 63884092dbcSRoman Gushchin return ret; 63984092dbcSRoman Gushchin } 64084092dbcSRoman Gushchin 641478b2784SMike Rapoport static int alloc_anon_50M_check_swap(const char *cgroup, void *arg) 642478b2784SMike Rapoport { 643478b2784SMike Rapoport long mem_max = (long)arg; 644478b2784SMike Rapoport size_t size = MB(50); 645478b2784SMike Rapoport char *buf, *ptr; 646478b2784SMike Rapoport long mem_current, swap_current; 647478b2784SMike Rapoport int ret = -1; 648478b2784SMike Rapoport 649478b2784SMike Rapoport buf = malloc(size); 650478b2784SMike Rapoport for (ptr = buf; ptr < buf + size; ptr += PAGE_SIZE) 651478b2784SMike Rapoport *ptr = 0; 652478b2784SMike Rapoport 653478b2784SMike Rapoport mem_current = cg_read_long(cgroup, "memory.current"); 654478b2784SMike Rapoport if (!mem_current || !values_close(mem_current, mem_max, 3)) 655478b2784SMike Rapoport goto cleanup; 656478b2784SMike Rapoport 657478b2784SMike Rapoport swap_current = cg_read_long(cgroup, "memory.swap.current"); 658478b2784SMike Rapoport if (!swap_current || 659478b2784SMike Rapoport !values_close(mem_current + swap_current, size, 3)) 660478b2784SMike Rapoport goto cleanup; 661478b2784SMike Rapoport 662478b2784SMike Rapoport ret = 0; 663478b2784SMike Rapoport cleanup: 664478b2784SMike Rapoport free(buf); 665478b2784SMike Rapoport return ret; 666478b2784SMike Rapoport } 667478b2784SMike Rapoport 668478b2784SMike Rapoport /* 669478b2784SMike Rapoport * This test checks that memory.swap.max limits the amount of 670478b2784SMike Rapoport * anonymous memory which can be swapped out. 671478b2784SMike Rapoport */ 672478b2784SMike Rapoport static int test_memcg_swap_max(const char *root) 673478b2784SMike Rapoport { 674478b2784SMike Rapoport int ret = KSFT_FAIL; 675478b2784SMike Rapoport char *memcg; 676478b2784SMike Rapoport long max; 677478b2784SMike Rapoport 678478b2784SMike Rapoport if (!is_swap_enabled()) 679478b2784SMike Rapoport return KSFT_SKIP; 680478b2784SMike Rapoport 681478b2784SMike Rapoport memcg = cg_name(root, "memcg_test"); 682478b2784SMike Rapoport if (!memcg) 683478b2784SMike Rapoport goto cleanup; 684478b2784SMike Rapoport 685478b2784SMike Rapoport if (cg_create(memcg)) 686478b2784SMike Rapoport goto cleanup; 687478b2784SMike Rapoport 688478b2784SMike Rapoport if (cg_read_long(memcg, "memory.swap.current")) { 689478b2784SMike Rapoport ret = KSFT_SKIP; 690478b2784SMike Rapoport goto cleanup; 691478b2784SMike Rapoport } 692478b2784SMike Rapoport 693478b2784SMike Rapoport if (cg_read_strcmp(memcg, "memory.max", "max\n")) 694478b2784SMike Rapoport goto cleanup; 695478b2784SMike Rapoport 696478b2784SMike Rapoport if (cg_read_strcmp(memcg, "memory.swap.max", "max\n")) 697478b2784SMike Rapoport goto cleanup; 698478b2784SMike Rapoport 699478b2784SMike Rapoport if (cg_write(memcg, "memory.swap.max", "30M")) 700478b2784SMike Rapoport goto cleanup; 701478b2784SMike Rapoport 702478b2784SMike Rapoport if (cg_write(memcg, "memory.max", "30M")) 703478b2784SMike Rapoport goto cleanup; 704478b2784SMike Rapoport 705478b2784SMike Rapoport /* Should be killed by OOM killer */ 706478b2784SMike Rapoport if (!cg_run(memcg, alloc_anon, (void *)MB(100))) 707478b2784SMike Rapoport goto cleanup; 708478b2784SMike Rapoport 709478b2784SMike Rapoport if (cg_read_key_long(memcg, "memory.events", "oom ") != 1) 710478b2784SMike Rapoport goto cleanup; 711478b2784SMike Rapoport 712478b2784SMike Rapoport if (cg_read_key_long(memcg, "memory.events", "oom_kill ") != 1) 713478b2784SMike Rapoport goto cleanup; 714478b2784SMike Rapoport 715478b2784SMike Rapoport if (cg_run(memcg, alloc_anon_50M_check_swap, (void *)MB(30))) 716478b2784SMike Rapoport goto cleanup; 717478b2784SMike Rapoport 718478b2784SMike Rapoport max = cg_read_key_long(memcg, "memory.events", "max "); 719478b2784SMike Rapoport if (max <= 0) 720478b2784SMike Rapoport goto cleanup; 721478b2784SMike Rapoport 722478b2784SMike Rapoport ret = KSFT_PASS; 723478b2784SMike Rapoport 724478b2784SMike Rapoport cleanup: 725478b2784SMike Rapoport cg_destroy(memcg); 726478b2784SMike Rapoport free(memcg); 727478b2784SMike Rapoport 728478b2784SMike Rapoport return ret; 729478b2784SMike Rapoport } 730478b2784SMike Rapoport 73184092dbcSRoman Gushchin /* 73284092dbcSRoman Gushchin * This test disables swapping and tries to allocate anonymous memory 73384092dbcSRoman Gushchin * up to OOM. Then it checks for oom and oom_kill events in 73484092dbcSRoman Gushchin * memory.events. 73584092dbcSRoman Gushchin */ 73684092dbcSRoman Gushchin static int test_memcg_oom_events(const char *root) 73784092dbcSRoman Gushchin { 73884092dbcSRoman Gushchin int ret = KSFT_FAIL; 73984092dbcSRoman Gushchin char *memcg; 74084092dbcSRoman Gushchin 74184092dbcSRoman Gushchin memcg = cg_name(root, "memcg_test"); 74284092dbcSRoman Gushchin if (!memcg) 74384092dbcSRoman Gushchin goto cleanup; 74484092dbcSRoman Gushchin 74584092dbcSRoman Gushchin if (cg_create(memcg)) 74684092dbcSRoman Gushchin goto cleanup; 74784092dbcSRoman Gushchin 74884092dbcSRoman Gushchin if (cg_write(memcg, "memory.max", "30M")) 74984092dbcSRoman Gushchin goto cleanup; 75084092dbcSRoman Gushchin 75184092dbcSRoman Gushchin if (cg_write(memcg, "memory.swap.max", "0")) 75284092dbcSRoman Gushchin goto cleanup; 75384092dbcSRoman Gushchin 75484092dbcSRoman Gushchin if (!cg_run(memcg, alloc_anon, (void *)MB(100))) 75584092dbcSRoman Gushchin goto cleanup; 75684092dbcSRoman Gushchin 75784092dbcSRoman Gushchin if (cg_read_strcmp(memcg, "cgroup.procs", "")) 75884092dbcSRoman Gushchin goto cleanup; 75984092dbcSRoman Gushchin 76084092dbcSRoman Gushchin if (cg_read_key_long(memcg, "memory.events", "oom ") != 1) 76184092dbcSRoman Gushchin goto cleanup; 76284092dbcSRoman Gushchin 76384092dbcSRoman Gushchin if (cg_read_key_long(memcg, "memory.events", "oom_kill ") != 1) 76484092dbcSRoman Gushchin goto cleanup; 76584092dbcSRoman Gushchin 76684092dbcSRoman Gushchin ret = KSFT_PASS; 76784092dbcSRoman Gushchin 76884092dbcSRoman Gushchin cleanup: 76984092dbcSRoman Gushchin cg_destroy(memcg); 77084092dbcSRoman Gushchin free(memcg); 77184092dbcSRoman Gushchin 77284092dbcSRoman Gushchin return ret; 77384092dbcSRoman Gushchin } 77484092dbcSRoman Gushchin 77584092dbcSRoman Gushchin #define T(x) { x, #x } 77684092dbcSRoman Gushchin struct memcg_test { 77784092dbcSRoman Gushchin int (*fn)(const char *root); 77884092dbcSRoman Gushchin const char *name; 77984092dbcSRoman Gushchin } tests[] = { 78084092dbcSRoman Gushchin T(test_memcg_subtree_control), 78184092dbcSRoman Gushchin T(test_memcg_current), 78284092dbcSRoman Gushchin T(test_memcg_min), 78384092dbcSRoman Gushchin T(test_memcg_low), 78484092dbcSRoman Gushchin T(test_memcg_high), 78584092dbcSRoman Gushchin T(test_memcg_max), 78684092dbcSRoman Gushchin T(test_memcg_oom_events), 787478b2784SMike Rapoport T(test_memcg_swap_max), 78884092dbcSRoman Gushchin }; 78984092dbcSRoman Gushchin #undef T 79084092dbcSRoman Gushchin 79184092dbcSRoman Gushchin int main(int argc, char **argv) 79284092dbcSRoman Gushchin { 79384092dbcSRoman Gushchin char root[PATH_MAX]; 79484092dbcSRoman Gushchin int i, ret = EXIT_SUCCESS; 79584092dbcSRoman Gushchin 79684092dbcSRoman Gushchin if (cg_find_unified_root(root, sizeof(root))) 79784092dbcSRoman Gushchin ksft_exit_skip("cgroup v2 isn't mounted\n"); 79884092dbcSRoman Gushchin 79984092dbcSRoman Gushchin /* 80084092dbcSRoman Gushchin * Check that memory controller is available: 80184092dbcSRoman Gushchin * memory is listed in cgroup.controllers 80284092dbcSRoman Gushchin */ 80384092dbcSRoman Gushchin if (cg_read_strstr(root, "cgroup.controllers", "memory")) 80484092dbcSRoman Gushchin ksft_exit_skip("memory controller isn't available\n"); 80584092dbcSRoman Gushchin 80684092dbcSRoman Gushchin for (i = 0; i < ARRAY_SIZE(tests); i++) { 80784092dbcSRoman Gushchin switch (tests[i].fn(root)) { 80884092dbcSRoman Gushchin case KSFT_PASS: 80984092dbcSRoman Gushchin ksft_test_result_pass("%s\n", tests[i].name); 81084092dbcSRoman Gushchin break; 81184092dbcSRoman Gushchin case KSFT_SKIP: 81284092dbcSRoman Gushchin ksft_test_result_skip("%s\n", tests[i].name); 81384092dbcSRoman Gushchin break; 81484092dbcSRoman Gushchin default: 81584092dbcSRoman Gushchin ret = EXIT_FAILURE; 81684092dbcSRoman Gushchin ksft_test_result_fail("%s\n", tests[i].name); 81784092dbcSRoman Gushchin break; 81884092dbcSRoman Gushchin } 81984092dbcSRoman Gushchin } 82084092dbcSRoman Gushchin 82184092dbcSRoman Gushchin return ret; 82284092dbcSRoman Gushchin } 823