1#!/bin/bash
2# SPDX-License-Identifier: GPL-2.0
3#
4# Test for cpuset v2 partition root state (PRS)
5#
6# The sched verbose flag is set, if available, so that the console log
7# can be examined for the correct setting of scheduling domain.
8#
9
10skip_test() {
11	echo "$1"
12	echo "Test SKIPPED"
13	exit 0
14}
15
16[[ $(id -u) -eq 0 ]] || skip_test "Test must be run as root!"
17
18# Set sched verbose flag, if available
19if [[ -d /sys/kernel/debug/sched ]]
20then
21	# Used to restore the original setting during cleanup
22	SCHED_DEBUG=$(cat /sys/kernel/debug/sched/verbose)
23	echo Y > /sys/kernel/debug/sched/verbose
24fi
25
26# Get wait_inotify location
27WAIT_INOTIFY=$(cd $(dirname $0); pwd)/wait_inotify
28
29# Find cgroup v2 mount point
30CGROUP2=$(mount -t cgroup2 | head -1 | awk -e '{print $3}')
31[[ -n "$CGROUP2" ]] || skip_test "Cgroup v2 mount point not found!"
32
33CPUS=$(lscpu | grep "^CPU(s):" | sed -e "s/.*:[[:space:]]*//")
34[[ $CPUS -lt 8 ]] && skip_test "Test needs at least 8 cpus available!"
35
36# Set verbose flag and delay factor
37PROG=$1
38VERBOSE=
39DELAY_FACTOR=1
40while [[ "$1" = -* ]]
41do
42	case "$1" in
43		-v) VERBOSE=1
44		    break
45		    ;;
46		-d) DELAY_FACTOR=$2
47		    shift
48		    break
49		    ;;
50		*)  echo "Usage: $PROG [-v] [-d <delay-factor>"
51		    exit
52		    ;;
53	esac
54	shift
55done
56
57cd $CGROUP2
58echo +cpuset > cgroup.subtree_control
59[[ -d test ]] || mkdir test
60cd test
61
62cleanup()
63{
64	online_cpus
65	rmdir A1/A2/A3 A1/A2 A1 B1 > /dev/null 2>&1
66	cd ..
67	rmdir test > /dev/null 2>&1
68	echo "$SCHED_DEBUG" > /sys/kernel/debug/sched/verbose
69}
70
71# Pause in ms
72pause()
73{
74	DELAY=$1
75	LOOP=0
76	while [[ $LOOP -lt $DELAY_FACTOR ]]
77	do
78		sleep $DELAY
79		((LOOP++))
80	done
81	return 0
82}
83
84console_msg()
85{
86	MSG=$1
87	echo "$MSG"
88	echo "" > /dev/console
89	echo "$MSG" > /dev/console
90	pause 0.01
91}
92
93test_partition()
94{
95	EXPECTED_VAL=$1
96	echo $EXPECTED_VAL > cpuset.cpus.partition
97	[[ $? -eq 0 ]] || exit 1
98	ACTUAL_VAL=$(cat cpuset.cpus.partition)
99	[[ $ACTUAL_VAL != $EXPECTED_VAL ]] && {
100		echo "cpuset.cpus.partition: expect $EXPECTED_VAL, found $EXPECTED_VAL"
101		echo "Test FAILED"
102		exit 1
103	}
104}
105
106test_effective_cpus()
107{
108	EXPECTED_VAL=$1
109	ACTUAL_VAL=$(cat cpuset.cpus.effective)
110	[[ "$ACTUAL_VAL" != "$EXPECTED_VAL" ]] && {
111		echo "cpuset.cpus.effective: expect '$EXPECTED_VAL', found '$EXPECTED_VAL'"
112		echo "Test FAILED"
113		exit 1
114	}
115}
116
117# Adding current process to cgroup.procs as a test
118test_add_proc()
119{
120	OUTSTR="$1"
121	ERRMSG=$((echo $$ > cgroup.procs) |& cat)
122	echo $ERRMSG | grep -q "$OUTSTR"
123	[[ $? -ne 0 ]] && {
124		echo "cgroup.procs: expect '$OUTSTR', got '$ERRMSG'"
125		echo "Test FAILED"
126		exit 1
127	}
128	echo $$ > $CGROUP2/cgroup.procs	# Move out the task
129}
130
131#
132# Testing the new "isolated" partition root type
133#
134test_isolated()
135{
136	echo 2-3 > cpuset.cpus
137	TYPE=$(cat cpuset.cpus.partition)
138	[[ $TYPE = member ]] || echo member > cpuset.cpus.partition
139
140	console_msg "Change from member to root"
141	test_partition root
142
143	console_msg "Change from root to isolated"
144	test_partition isolated
145
146	console_msg "Change from isolated to member"
147	test_partition member
148
149	console_msg "Change from member to isolated"
150	test_partition isolated
151
152	console_msg "Change from isolated to root"
153	test_partition root
154
155	console_msg "Change from root to member"
156	test_partition member
157
158	#
159	# Testing partition root with no cpu
160	#
161	console_msg "Distribute all cpus to child partition"
162	echo +cpuset > cgroup.subtree_control
163	test_partition root
164
165	mkdir A1
166	cd A1
167	echo 2-3 > cpuset.cpus
168	test_partition root
169	test_effective_cpus 2-3
170	cd ..
171	test_effective_cpus ""
172
173	console_msg "Moving task to partition test"
174	test_add_proc "No space left"
175	cd A1
176	test_add_proc ""
177	cd ..
178
179	console_msg "Shrink and expand child partition"
180	cd A1
181	echo 2 > cpuset.cpus
182	cd ..
183	test_effective_cpus 3
184	cd A1
185	echo 2-3 > cpuset.cpus
186	cd ..
187	test_effective_cpus ""
188
189	# Cleaning up
190	console_msg "Cleaning up"
191	echo $$ > $CGROUP2/cgroup.procs
192	[[ -d A1 ]] && rmdir A1
193}
194
195#
196# Cpuset controller state transition test matrix.
197#
198# Cgroup test hierarchy
199#
200# test -- A1 -- A2 -- A3
201#      \- B1
202#
203#  P<v> = set cpus.partition (0:member, 1:root, 2:isolated, -1:root invalid)
204#  C<l> = add cpu-list
205#  S<p> = use prefix in subtree_control
206#  T    = put a task into cgroup
207#  O<c>-<v> = Write <v> to CPU online file of <c>
208#
209SETUP_A123_PARTITIONS="C1-3:P1:S+ C2-3:P1:S+ C3:P1"
210TEST_MATRIX=(
211	# test  old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate
212	# ----  ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------
213	"  S+    C0-1     .      .    C2-3    S+    C4-5     .      .     0 A2:0-1"
214	"  S+    C0-1     .      .    C2-3    P1      .      .      .     0 "
215	"  S+    C0-1     .      .    C2-3   P1:S+ C0-1:P1   .      .     0 "
216	"  S+    C0-1     .      .    C2-3   P1:S+  C1:P1    .      .     0 "
217	"  S+   C0-1:S+   .      .    C2-3     .      .      .     P1     0 "
218	"  S+   C0-1:P1   .      .    C2-3    S+     C1      .      .     0 "
219	"  S+   C0-1:P1   .      .    C2-3    S+    C1:P1    .      .     0 "
220	"  S+   C0-1:P1   .      .    C2-3    S+    C1:P1    .     P1     0 "
221	"  S+   C0-1:P1   .      .    C2-3   C4-5     .      .      .     0 A1:4-5"
222	"  S+   C0-1:P1   .      .    C2-3  S+:C4-5   .      .      .     0 A1:4-5"
223	"  S+    C0-1     .      .   C2-3:P1   .      .      .     C2     0 "
224	"  S+    C0-1     .      .   C2-3:P1   .      .      .    C4-5    0 B1:4-5"
225	"  S+ C0-3:P1:S+ C2-3:P1 .      .      .      .      .      .     0 A1:0-1,A2:2-3"
226	"  S+ C0-3:P1:S+ C2-3:P1 .      .     C1-3    .      .      .     0 A1:1,A2:2-3"
227	"  S+ C2-3:P1:S+  C3:P1  .      .     C3      .      .      .     0 A1:,A2:3 A1:P1,A2:P1"
228	"  S+ C2-3:P1:S+  C3:P1  .      .     C3      P0     .      .     0 A1:3,A2:3 A1:P1,A2:P0"
229	"  S+ C2-3:P1:S+  C2:P1  .      .     C2-4    .      .      .     0 A1:3-4,A2:2"
230	"  S+ C2-3:P1:S+  C3:P1  .      .     C3      .      .     C0-2   0 A1:,B1:0-2 A1:P1,A2:P1"
231	"  S+ $SETUP_A123_PARTITIONS    .     C2-3    .      .      .     0 A1:,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
232
233	# CPU offlining cases:
234	"  S+    C0-1     .      .    C2-3    S+    C4-5     .     O2-0   0 A1:0-1,B1:3"
235	"  S+ C0-3:P1:S+ C2-3:P1 .      .     O2-0    .      .      .     0 A1:0-1,A2:3"
236	"  S+ C0-3:P1:S+ C2-3:P1 .      .     O2-0   O2-1    .      .     0 A1:0-1,A2:2-3"
237	"  S+ C0-3:P1:S+ C2-3:P1 .      .     O1-0    .      .      .     0 A1:0,A2:2-3"
238	"  S+ C0-3:P1:S+ C2-3:P1 .      .     O1-0   O1-1    .      .     0 A1:0-1,A2:2-3"
239	"  S+ C2-3:P1:S+  C3:P1  .      .     O3-0   O3-1    .      .     0 A1:2,A2:3 A1:P1,A2:P1"
240	"  S+ C2-3:P1:S+  C3:P2  .      .     O3-0   O3-1    .      .     0 A1:2,A2:3 A1:P1,A2:P2"
241	"  S+ C2-3:P1:S+  C3:P1  .      .     O2-0   O2-1    .      .     0 A1:2,A2:3 A1:P1,A2:P1"
242	"  S+ C2-3:P1:S+  C3:P2  .      .     O2-0   O2-1    .      .     0 A1:2,A2:3 A1:P1,A2:P2"
243	"  S+ C2-3:P1:S+  C3:P1  .      .     O2-0    .      .      .     0 A1:,A2:3 A1:P1,A2:P1"
244	"  S+ C2-3:P1:S+  C3:P1  .      .     O3-0    .      .      .     0 A1:2,A2: A1:P1,A2:P1"
245	"  S+ C2-3:P1:S+  C3:P1  .      .    T:O2-0   .      .      .     0 A1:3,A2:3 A1:P1,A2:P-1"
246	"  S+ C2-3:P1:S+  C3:P1  .      .      .    T:O3-0   .      .     0 A1:2,A2:2 A1:P1,A2:P-1"
247	"  S+ $SETUP_A123_PARTITIONS    .     O1-0    .      .      .     0 A1:,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
248	"  S+ $SETUP_A123_PARTITIONS    .     O2-0    .      .      .     0 A1:1,A2:,A3:3 A1:P1,A2:P1,A3:P1"
249	"  S+ $SETUP_A123_PARTITIONS    .     O3-0    .      .      .     0 A1:1,A2:2,A3: A1:P1,A2:P1,A3:P1"
250	"  S+ $SETUP_A123_PARTITIONS    .    T:O1-0   .      .      .     0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1"
251	"  S+ $SETUP_A123_PARTITIONS    .      .    T:O2-0   .      .     0 A1:1,A2:3,A3:3 A1:P1,A2:P1,A3:P-1"
252	"  S+ $SETUP_A123_PARTITIONS    .      .      .    T:O3-0   .     0 A1:1,A2:2,A3:2 A1:P1,A2:P1,A3:P-1"
253	"  S+ $SETUP_A123_PARTITIONS    .    T:O1-0  O1-1    .      .     0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
254	"  S+ $SETUP_A123_PARTITIONS    .      .    T:O2-0  O2-1    .     0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
255	"  S+ $SETUP_A123_PARTITIONS    .      .      .    T:O3-0  O3-1   0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
256	"  S+ $SETUP_A123_PARTITIONS    .    T:O1-0  O2-0   O1-1    .     0 A1:1,A2:,A3:3 A1:P1,A2:P1,A3:P1"
257	"  S+ $SETUP_A123_PARTITIONS    .    T:O1-0  O2-0   O2-1    .     0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1"
258
259	# test  old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate
260	# ----  ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------
261	#
262	# Incorrect change to cpuset.cpus invalidates partition root
263	#
264	# Adding CPUs to partition root that are not in parent's
265	# cpuset.cpus is allowed, but those extra CPUs are ignored.
266	"  S+ C2-3:P1:S+ C3:P1   .      .      .     C2-4    .      .     0 A1:,A2:2-3 A1:P1,A2:P1"
267
268	# Taking away all CPUs from parent or itself if there are tasks
269	# will make the partition invalid.
270	"  S+ C2-3:P1:S+  C3:P1  .      .      T     C2-3    .      .     0 A1:2-3,A2:2-3 A1:P1,A2:P-1"
271	"  S+ $SETUP_A123_PARTITIONS    .    T:C2-3   .      .      .     0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P-1,A3:P-1"
272	"  S+ $SETUP_A123_PARTITIONS    . T:C2-3:C1-3 .      .      .     0 A1:1,A2:2,A3:3 A1:P1,A2:P1,A3:P1"
273
274	# Changing a partition root to member makes child partitions invalid
275	"  S+ C2-3:P1:S+  C3:P1  .      .      P0     .      .      .     0 A1:2-3,A2:3 A1:P0,A2:P-1"
276	"  S+ $SETUP_A123_PARTITIONS    .     C2-3    P0     .      .     0 A1:2-3,A2:2-3,A3:3 A1:P1,A2:P0,A3:P-1"
277
278	# cpuset.cpus can contains cpus not in parent's cpuset.cpus as long
279	# as they overlap.
280	"  S+ C2-3:P1:S+  .      .      .      .   C3-4:P1   .      .     0 A1:2,A2:3 A1:P1,A2:P1"
281
282	# Deletion of CPUs distributed to child cgroup is allowed.
283	"  S+ C0-1:P1:S+ C1      .    C2-3   C4-5     .      .      .     0 A1:4-5,A2:4-5"
284
285	# To become a valid partition root, cpuset.cpus must overlap parent's
286	# cpuset.cpus.
287	"  S+   C0-1:P1   .      .    C2-3    S+   C4-5:P1   .      .     0 A1:0-1,A2:0-1 A1:P1,A2:P-1"
288
289	# Enabling partition with child cpusets is allowed
290	"  S+   C0-1:S+  C1      .    C2-3    P1      .      .      .     0 A1:0-1,A2:1 A1:P1"
291
292	# A partition root with non-partition root parent is invalid, but it
293	# can be made valid if its parent becomes a partition root too.
294	"  S+   C0-1:S+  C1      .    C2-3     .      P2     .      .     0 A1:0-1,A2:1 A1:P0,A2:P-2"
295	"  S+   C0-1:S+ C1:P2    .    C2-3     P1     .      .      .     0 A1:0,A2:1 A1:P1,A2:P2"
296
297	# A non-exclusive cpuset.cpus change will invalidate partition and its siblings
298	"  S+   C0-1:P1   .      .    C2-3   C0-2     .      .      .     0 A1:0-2,B1:2-3 A1:P-1,B1:P0"
299	"  S+   C0-1:P1   .      .  P1:C2-3  C0-2   .      .      .     0 A1:0-2,B1:2-3 A1:P-1,B1:P-1"
300	"  S+    C0-1     .      .  P1:C2-3  C0-2   .      .      .     0 A1:0-2,B1:2-3 A1:P0,B1:P-1"
301
302	# test  old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate
303	# ----  ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------
304	# Failure cases:
305
306	# A task cannot be added to a partition with no cpu
307	"  S+ C2-3:P1:S+  C3:P1  .      .    O2-0:T   .      .      .     1 A1:,A2:3 A1:P1,A2:P1"
308)
309
310#
311# Write to the cpu online file
312#  $1 - <c>-<v> where <c> = cpu number, <v> value to be written
313#
314write_cpu_online()
315{
316	CPU=${1%-*}
317	VAL=${1#*-}
318	CPUFILE=//sys/devices/system/cpu/cpu${CPU}/online
319	if [[ $VAL -eq 0 ]]
320	then
321		OFFLINE_CPUS="$OFFLINE_CPUS $CPU"
322	else
323		[[ -n "$OFFLINE_CPUS" ]] && {
324			OFFLINE_CPUS=$(echo $CPU $CPU $OFFLINE_CPUS | fmt -1 |\
325					sort | uniq -u)
326		}
327	fi
328	echo $VAL > $CPUFILE
329	pause 0.01
330}
331
332#
333# Set controller state
334#  $1 - cgroup directory
335#  $2 - state
336#  $3 - showerr
337#
338# The presence of ":" in state means transition from one to the next.
339#
340set_ctrl_state()
341{
342	TMPMSG=/tmp/.msg_$$
343	CGRP=$1
344	STATE=$2
345	SHOWERR=${3}${VERBOSE}
346	CTRL=${CTRL:=$CONTROLLER}
347	HASERR=0
348	REDIRECT="2> $TMPMSG"
349	[[ -z "$STATE" || "$STATE" = '.' ]] && return 0
350
351	rm -f $TMPMSG
352	for CMD in $(echo $STATE | sed -e "s/:/ /g")
353	do
354		TFILE=$CGRP/cgroup.procs
355		SFILE=$CGRP/cgroup.subtree_control
356		PFILE=$CGRP/cpuset.cpus.partition
357		CFILE=$CGRP/cpuset.cpus
358		S=$(expr substr $CMD 1 1)
359		if [[ $S = S ]]
360		then
361			PREFIX=${CMD#?}
362			COMM="echo ${PREFIX}${CTRL} > $SFILE"
363			eval $COMM $REDIRECT
364		elif [[ $S = C ]]
365		then
366			CPUS=${CMD#?}
367			COMM="echo $CPUS > $CFILE"
368			eval $COMM $REDIRECT
369		elif [[ $S = P ]]
370		then
371			VAL=${CMD#?}
372			case $VAL in
373			0)  VAL=member
374			    ;;
375			1)  VAL=root
376			    ;;
377			2)  VAL=isolated
378			    ;;
379			*)
380			    echo "Invalid partition state - $VAL"
381			    exit 1
382			    ;;
383			esac
384			COMM="echo $VAL > $PFILE"
385			eval $COMM $REDIRECT
386		elif [[ $S = O ]]
387		then
388			VAL=${CMD#?}
389			write_cpu_online $VAL
390		elif [[ $S = T ]]
391		then
392			COMM="echo 0 > $TFILE"
393			eval $COMM $REDIRECT
394		fi
395		RET=$?
396		[[ $RET -ne 0 ]] && {
397			[[ -n "$SHOWERR" ]] && {
398				echo "$COMM"
399				cat $TMPMSG
400			}
401			HASERR=1
402		}
403		pause 0.01
404		rm -f $TMPMSG
405	done
406	return $HASERR
407}
408
409set_ctrl_state_noerr()
410{
411	CGRP=$1
412	STATE=$2
413	[[ -d $CGRP ]] || mkdir $CGRP
414	set_ctrl_state $CGRP $STATE 1
415	[[ $? -ne 0 ]] && {
416		echo "ERROR: Failed to set $2 to cgroup $1!"
417		exit 1
418	}
419}
420
421online_cpus()
422{
423	[[ -n "OFFLINE_CPUS" ]] && {
424		for C in $OFFLINE_CPUS
425		do
426			write_cpu_online ${C}-1
427		done
428	}
429}
430
431#
432# Return 1 if the list of effective cpus isn't the same as the initial list.
433#
434reset_cgroup_states()
435{
436	echo 0 > $CGROUP2/cgroup.procs
437	online_cpus
438	rmdir A1/A2/A3 A1/A2 A1 B1 > /dev/null 2>&1
439	set_ctrl_state . S-
440	pause 0.01
441}
442
443dump_states()
444{
445	for DIR in A1 A1/A2 A1/A2/A3 B1
446	do
447		ECPUS=$DIR/cpuset.cpus.effective
448		PRS=$DIR/cpuset.cpus.partition
449		[[ -e $ECPUS ]] && echo "$ECPUS: $(cat $ECPUS)"
450		[[ -e $PRS   ]] && echo "$PRS: $(cat $PRS)"
451	done
452}
453
454#
455# Check effective cpus
456# $1 - check string, format: <cgroup>:<cpu-list>[,<cgroup>:<cpu-list>]*
457#
458check_effective_cpus()
459{
460	CHK_STR=$1
461	for CHK in $(echo $CHK_STR | sed -e "s/,/ /g")
462	do
463		set -- $(echo $CHK | sed -e "s/:/ /g")
464		CGRP=$1
465		CPUS=$2
466		[[ $CGRP = A2 ]] && CGRP=A1/A2
467		[[ $CGRP = A3 ]] && CGRP=A1/A2/A3
468		FILE=$CGRP/cpuset.cpus.effective
469		[[ -e $FILE ]] || return 1
470		[[ $CPUS = $(cat $FILE) ]] || return 1
471	done
472}
473
474#
475# Check cgroup states
476#  $1 - check string, format: <cgroup>:<state>[,<cgroup>:<state>]*
477#
478check_cgroup_states()
479{
480	CHK_STR=$1
481	for CHK in $(echo $CHK_STR | sed -e "s/,/ /g")
482	do
483		set -- $(echo $CHK | sed -e "s/:/ /g")
484		CGRP=$1
485		STATE=$2
486		FILE=
487		EVAL=$(expr substr $STATE 2 2)
488		[[ $CGRP = A2 ]] && CGRP=A1/A2
489		[[ $CGRP = A3 ]] && CGRP=A1/A2/A3
490
491		case $STATE in
492			P*) FILE=$CGRP/cpuset.cpus.partition
493			    ;;
494			*)  echo "Unknown state: $STATE!"
495			    exit 1
496			    ;;
497		esac
498		VAL=$(cat $FILE)
499
500		case "$VAL" in
501			member) VAL=0
502				;;
503			root)	VAL=1
504				;;
505			isolated)
506				VAL=2
507				;;
508			"root invalid"*)
509				VAL=-1
510				;;
511			"isolated invalid"*)
512				VAL=-2
513				;;
514		esac
515		[[ $EVAL != $VAL ]] && return 1
516	done
517	return 0
518}
519
520#
521# Run cpuset state transition test
522#  $1 - test matrix name
523#
524# This test is somewhat fragile as delays (sleep x) are added in various
525# places to make sure state changes are fully propagated before the next
526# action. These delays may need to be adjusted if running in a slower machine.
527#
528run_state_test()
529{
530	TEST=$1
531	CONTROLLER=cpuset
532	CPULIST=0-6
533	I=0
534	eval CNT="\${#$TEST[@]}"
535
536	reset_cgroup_states
537	echo $CPULIST > cpuset.cpus
538	echo root > cpuset.cpus.partition
539	console_msg "Running state transition test ..."
540
541	while [[ $I -lt $CNT ]]
542	do
543		echo "Running test $I ..." > /dev/console
544		eval set -- "\${$TEST[$I]}"
545		ROOT=$1
546		OLD_A1=$2
547		OLD_A2=$3
548		OLD_A3=$4
549		OLD_B1=$5
550		NEW_A1=$6
551		NEW_A2=$7
552		NEW_A3=$8
553		NEW_B1=$9
554		RESULT=${10}
555		ECPUS=${11}
556		STATES=${12}
557
558		set_ctrl_state_noerr .        $ROOT
559		set_ctrl_state_noerr A1       $OLD_A1
560		set_ctrl_state_noerr A1/A2    $OLD_A2
561		set_ctrl_state_noerr A1/A2/A3 $OLD_A3
562		set_ctrl_state_noerr B1       $OLD_B1
563		RETVAL=0
564		set_ctrl_state A1       $NEW_A1; ((RETVAL += $?))
565		set_ctrl_state A1/A2    $NEW_A2; ((RETVAL += $?))
566		set_ctrl_state A1/A2/A3 $NEW_A3; ((RETVAL += $?))
567		set_ctrl_state B1       $NEW_B1; ((RETVAL += $?))
568
569		[[ $RETVAL -ne $RESULT ]] && {
570			echo "Test $TEST[$I] failed result check!"
571			eval echo \"\${$TEST[$I]}\"
572			dump_states
573			online_cpus
574			exit 1
575		}
576
577		[[ -n "$ECPUS" && "$ECPUS" != . ]] && {
578			check_effective_cpus $ECPUS
579			[[ $? -ne 0 ]] && {
580				echo "Test $TEST[$I] failed effective CPU check!"
581				eval echo \"\${$TEST[$I]}\"
582				echo
583				dump_states
584				online_cpus
585				exit 1
586			}
587		}
588
589		[[ -n "$STATES" ]] && {
590			check_cgroup_states $STATES
591			[[ $? -ne 0 ]] && {
592				echo "FAILED: Test $TEST[$I] failed states check!"
593				eval echo \"\${$TEST[$I]}\"
594				echo
595				dump_states
596				online_cpus
597				exit 1
598			}
599		}
600
601		reset_cgroup_states
602		#
603		# Check to see if effective cpu list changes
604		#
605		pause 0.05
606		NEWLIST=$(cat cpuset.cpus.effective)
607		[[ $NEWLIST != $CPULIST ]] && {
608			echo "Effective cpus changed to $NEWLIST after test $I!"
609			exit 1
610		}
611		[[ -n "$VERBOSE" ]] && echo "Test $I done."
612		((I++))
613	done
614	echo "All $I tests of $TEST PASSED."
615
616	echo member > cpuset.cpus.partition
617}
618
619#
620# Wait for inotify event for the given file and read it
621# $1: cgroup file to wait for
622# $2: file to store the read result
623#
624wait_inotify()
625{
626	CGROUP_FILE=$1
627	OUTPUT_FILE=$2
628
629	$WAIT_INOTIFY $CGROUP_FILE
630	cat $CGROUP_FILE > $OUTPUT_FILE
631}
632
633#
634# Test if inotify events are properly generated when going into and out of
635# invalid partition state.
636#
637test_inotify()
638{
639	ERR=0
640	PRS=/tmp/.prs_$$
641	[[ -f $WAIT_INOTIFY ]] || {
642		echo "wait_inotify not found, inotify test SKIPPED."
643		return
644	}
645
646	pause 0.01
647	echo 1 > cpuset.cpus
648	echo 0 > cgroup.procs
649	echo root > cpuset.cpus.partition
650	pause 0.01
651	rm -f $PRS
652	wait_inotify $PWD/cpuset.cpus.partition $PRS &
653	pause 0.01
654	set_ctrl_state . "O1-0"
655	pause 0.01
656	check_cgroup_states ".:P-1"
657	if [[ $? -ne 0 ]]
658	then
659		echo "FAILED: Inotify test - partition not invalid"
660		ERR=1
661	elif [[ ! -f $PRS ]]
662	then
663		echo "FAILED: Inotify test - event not generated"
664		ERR=1
665		kill %1
666	elif [[ $(cat $PRS) != "root invalid"* ]]
667	then
668		echo "FAILED: Inotify test - incorrect state"
669		cat $PRS
670		ERR=1
671	fi
672	online_cpus
673	echo member > cpuset.cpus.partition
674	echo 0 > ../cgroup.procs
675	if [[ $ERR -ne 0 ]]
676	then
677		exit 1
678	else
679		echo "Inotify test PASSED"
680	fi
681}
682
683trap cleanup 0 2 3 6
684run_state_test TEST_MATRIX
685test_isolated
686test_inotify
687echo "All tests PASSED."
688cd ..
689rmdir test
690