13 static bool bpf_verifier_log_attr_valid(const struct bpf_verifier_log *log)  in bpf_verifier_log_attr_valid()  argument
16 	if (!!log->ubuf != !!log->len_total)  in bpf_verifier_log_attr_valid()
19 	if (log->ubuf && log->level == 0)  in bpf_verifier_log_attr_valid()
21 	if (log->level & ~BPF_LOG_MASK)  in bpf_verifier_log_attr_valid()
23 	if (log->len_total > UINT_MAX >> 2)  in bpf_verifier_log_attr_valid()
28 int bpf_vlog_init(struct bpf_verifier_log *log, u32 log_level,  in bpf_vlog_init()  argument
31 	log->level = log_level;  in bpf_vlog_init()
32 	log->ubuf = log_buf;  in bpf_vlog_init()
33 	log->len_total = log_size;  in bpf_vlog_init()
36 	if (!bpf_verifier_log_attr_valid(log))  in bpf_vlog_init()
42 static void bpf_vlog_update_len_max(struct bpf_verifier_log *log, u32 add_len)  in bpf_vlog_update_len_max()  argument
45 	u64 len = log->end_pos + add_len;  in bpf_vlog_update_len_max()
52 		log->len_max = UINT_MAX;  in bpf_vlog_update_len_max()
53 	else if (len > log->len_max)  in bpf_vlog_update_len_max()
54 		log->len_max = len;  in bpf_vlog_update_len_max()
57 void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt,  in bpf_verifier_vlog()  argument
63 	n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args);  in bpf_verifier_vlog()
65 	if (log->level == BPF_LOG_KERNEL) {  in bpf_verifier_vlog()
66 		bool newline = n > 0 && log->kbuf[n - 1] == '\n';  in bpf_verifier_vlog()
68 		pr_err("BPF: %s%s", log->kbuf, newline ? "" : "\n");  in bpf_verifier_vlog()
73 	bpf_vlog_update_len_max(log, n);  in bpf_verifier_vlog()
75 	if (log->level & BPF_LOG_FIXED) {  in bpf_verifier_vlog()
78 		if (log->end_pos < log->len_total) {  in bpf_verifier_vlog()
79 			new_n = min_t(u32, log->len_total - log->end_pos, n);  in bpf_verifier_vlog()
80 			log->kbuf[new_n - 1] = '\0';  in bpf_verifier_vlog()
83 		cur_pos = log->end_pos;  in bpf_verifier_vlog()
84 		log->end_pos += n - 1; /* don't count terminating '\0' */  in bpf_verifier_vlog()
86 		if (log->ubuf && new_n &&  in bpf_verifier_vlog()
87 		    copy_to_user(log->ubuf + cur_pos, log->kbuf, new_n))  in bpf_verifier_vlog()
93 		new_end = log->end_pos + n;  in bpf_verifier_vlog()
94 		if (new_end - log->start_pos >= log->len_total)  in bpf_verifier_vlog()
95 			new_start = new_end - log->len_total;  in bpf_verifier_vlog()
97 			new_start = log->start_pos;  in bpf_verifier_vlog()
99 		log->start_pos = new_start;  in bpf_verifier_vlog()
100 		log->end_pos = new_end - 1; /* don't count terminating '\0' */  in bpf_verifier_vlog()
102 		if (!log->ubuf)  in bpf_verifier_vlog()
105 		new_n = min(n, log->len_total);  in bpf_verifier_vlog()
107 		div_u64_rem(cur_pos, log->len_total, &buf_start);  in bpf_verifier_vlog()
108 		div_u64_rem(new_end, log->len_total, &buf_end);  in bpf_verifier_vlog()
114 			buf_end = log->len_total;  in bpf_verifier_vlog()
124 			if (copy_to_user(log->ubuf + buf_start,  in bpf_verifier_vlog()
125 					 log->kbuf + n - new_n,  in bpf_verifier_vlog()
130 			if (copy_to_user(log->ubuf + buf_start,  in bpf_verifier_vlog()
131 					 log->kbuf + n - new_n,  in bpf_verifier_vlog()
132 					 log->len_total - buf_start))  in bpf_verifier_vlog()
134 			if (copy_to_user(log->ubuf,  in bpf_verifier_vlog()
135 					 log->kbuf + n - buf_end,  in bpf_verifier_vlog()
143 	log->ubuf = NULL;  in bpf_verifier_vlog()
146 void bpf_vlog_reset(struct bpf_verifier_log *log, u64 new_pos)  in bpf_vlog_reset()  argument
151 	if (WARN_ON_ONCE(new_pos > log->end_pos))  in bpf_vlog_reset()
154 	if (!bpf_verifier_log_needed(log) || log->level == BPF_LOG_KERNEL)  in bpf_vlog_reset()
161 	log->end_pos = new_pos;  in bpf_vlog_reset()
162 	if (log->end_pos < log->start_pos)  in bpf_vlog_reset()
163 		log->start_pos = log->end_pos;  in bpf_vlog_reset()
165 	if (!log->ubuf)  in bpf_vlog_reset()
168 	if (log->level & BPF_LOG_FIXED)  in bpf_vlog_reset()
169 		pos = log->end_pos + 1;  in bpf_vlog_reset()
171 		div_u64_rem(new_pos, log->len_total, &pos);  in bpf_vlog_reset()
173 	if (pos < log->len_total && put_user(zero, log->ubuf + pos))  in bpf_vlog_reset()
174 		log->ubuf = NULL;  in bpf_vlog_reset()
185 static int bpf_vlog_reverse_ubuf(struct bpf_verifier_log *log, int start, int end)  in bpf_vlog_reverse_ubuf()  argument
188 	int n = sizeof(log->kbuf) / 2, nn;  in bpf_vlog_reverse_ubuf()
189 	char *lbuf = log->kbuf, *rbuf = log->kbuf + n;  in bpf_vlog_reverse_ubuf()
200 		if (copy_from_user(lbuf, log->ubuf + start, nn))  in bpf_vlog_reverse_ubuf()
202 		if (copy_from_user(rbuf, log->ubuf + end - nn, nn))  in bpf_vlog_reverse_ubuf()
211 		if (copy_to_user(log->ubuf + start, rbuf, nn))  in bpf_vlog_reverse_ubuf()
213 		if (copy_to_user(log->ubuf + end - nn, lbuf, nn))  in bpf_vlog_reverse_ubuf()
223 int bpf_vlog_finalize(struct bpf_verifier_log *log, u32 *log_size_actual)  in bpf_vlog_finalize()  argument
229 	if (!log || log->level == 0 || log->level == BPF_LOG_KERNEL)  in bpf_vlog_finalize()
232 	if (!log->ubuf)  in bpf_vlog_finalize()
235 	if (log->start_pos == 0)  in bpf_vlog_finalize()
271 	div_u64_rem(log->start_pos, log->len_total, &sublen);  in bpf_vlog_finalize()
272 	sublen = log->len_total - sublen;  in bpf_vlog_finalize()
274 	err = bpf_vlog_reverse_ubuf(log, 0, log->len_total);  in bpf_vlog_finalize()
275 	err = err ?: bpf_vlog_reverse_ubuf(log, 0, sublen);  in bpf_vlog_finalize()
276 	err = err ?: bpf_vlog_reverse_ubuf(log, sublen, log->len_total);  in bpf_vlog_finalize()
278 		log->ubuf = NULL;  in bpf_vlog_finalize()
281 	*log_size_actual = log->len_max;  in bpf_vlog_finalize()
287 	if (!!log->ubuf != !!log->len_total)  in bpf_vlog_finalize()
291 	if (log->ubuf && log->len_max > log->len_total)  in bpf_vlog_finalize()
306 	if (!bpf_verifier_log_needed(&env->log))  in bpf_verifier_log_write()
310 	bpf_verifier_vlog(&env->log, fmt, args);  in bpf_verifier_log_write()
315 __printf(2, 3) void bpf_log(struct bpf_verifier_log *log,  in bpf_log()  argument
320 	if (!bpf_verifier_log_needed(log))  in bpf_log()
324 	bpf_verifier_vlog(log, fmt, args);  in bpf_log()