xref: /openbmc/linux/drivers/char/random.c (revision 0e17c50f)
1 /*
2  * random.c -- A strong random number generator
3  *
4  * Copyright (C) 2017 Jason A. Donenfeld <Jason@zx2c4.com>. All
5  * Rights Reserved.
6  *
7  * Copyright Matt Mackall <mpm@selenic.com>, 2003, 2004, 2005
8  *
9  * Copyright Theodore Ts'o, 1994, 1995, 1996, 1997, 1998, 1999.  All
10  * rights reserved.
11  *
12  * Redistribution and use in source and binary forms, with or without
13  * modification, are permitted provided that the following conditions
14  * are met:
15  * 1. Redistributions of source code must retain the above copyright
16  *    notice, and the entire permission notice in its entirety,
17  *    including the disclaimer of warranties.
18  * 2. Redistributions in binary form must reproduce the above copyright
19  *    notice, this list of conditions and the following disclaimer in the
20  *    documentation and/or other materials provided with the distribution.
21  * 3. The name of the author may not be used to endorse or promote
22  *    products derived from this software without specific prior
23  *    written permission.
24  *
25  * ALTERNATIVELY, this product may be distributed under the terms of
26  * the GNU General Public License, in which case the provisions of the GPL are
27  * required INSTEAD OF the above restrictions.  (This clause is
28  * necessary due to a potential bad interaction between the GPL and
29  * the restrictions contained in a BSD-style copyright.)
30  *
31  * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
32  * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
33  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
34  * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
35  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
36  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
37  * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
38  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
39  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
41  * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
42  * DAMAGE.
43  */
44 
45 /*
46  * (now, with legal B.S. out of the way.....)
47  *
48  * This routine gathers environmental noise from device drivers, etc.,
49  * and returns good random numbers, suitable for cryptographic use.
50  * Besides the obvious cryptographic uses, these numbers are also good
51  * for seeding TCP sequence numbers, and other places where it is
52  * desirable to have numbers which are not only random, but hard to
53  * predict by an attacker.
54  *
55  * Theory of operation
56  * ===================
57  *
58  * Computers are very predictable devices.  Hence it is extremely hard
59  * to produce truly random numbers on a computer --- as opposed to
60  * pseudo-random numbers, which can easily generated by using a
61  * algorithm.  Unfortunately, it is very easy for attackers to guess
62  * the sequence of pseudo-random number generators, and for some
63  * applications this is not acceptable.  So instead, we must try to
64  * gather "environmental noise" from the computer's environment, which
65  * must be hard for outside attackers to observe, and use that to
66  * generate random numbers.  In a Unix environment, this is best done
67  * from inside the kernel.
68  *
69  * Sources of randomness from the environment include inter-keyboard
70  * timings, inter-interrupt timings from some interrupts, and other
71  * events which are both (a) non-deterministic and (b) hard for an
72  * outside observer to measure.  Randomness from these sources are
73  * added to an "entropy pool", which is mixed using a CRC-like function.
74  * This is not cryptographically strong, but it is adequate assuming
75  * the randomness is not chosen maliciously, and it is fast enough that
76  * the overhead of doing it on every interrupt is very reasonable.
77  * As random bytes are mixed into the entropy pool, the routines keep
78  * an *estimate* of how many bits of randomness have been stored into
79  * the random number generator's internal state.
80  *
81  * When random bytes are desired, they are obtained by taking the SHA
82  * hash of the contents of the "entropy pool".  The SHA hash avoids
83  * exposing the internal state of the entropy pool.  It is believed to
84  * be computationally infeasible to derive any useful information
85  * about the input of SHA from its output.  Even if it is possible to
86  * analyze SHA in some clever way, as long as the amount of data
87  * returned from the generator is less than the inherent entropy in
88  * the pool, the output data is totally unpredictable.  For this
89  * reason, the routine decreases its internal estimate of how many
90  * bits of "true randomness" are contained in the entropy pool as it
91  * outputs random numbers.
92  *
93  * If this estimate goes to zero, the routine can still generate
94  * random numbers; however, an attacker may (at least in theory) be
95  * able to infer the future output of the generator from prior
96  * outputs.  This requires successful cryptanalysis of SHA, which is
97  * not believed to be feasible, but there is a remote possibility.
98  * Nonetheless, these numbers should be useful for the vast majority
99  * of purposes.
100  *
101  * Exported interfaces ---- output
102  * ===============================
103  *
104  * There are four exported interfaces; two for use within the kernel,
105  * and two or use from userspace.
106  *
107  * Exported interfaces ---- userspace output
108  * -----------------------------------------
109  *
110  * The userspace interfaces are two character devices /dev/random and
111  * /dev/urandom.  /dev/random is suitable for use when very high
112  * quality randomness is desired (for example, for key generation or
113  * one-time pads), as it will only return a maximum of the number of
114  * bits of randomness (as estimated by the random number generator)
115  * contained in the entropy pool.
116  *
117  * The /dev/urandom device does not have this limit, and will return
118  * as many bytes as are requested.  As more and more random bytes are
119  * requested without giving time for the entropy pool to recharge,
120  * this will result in random numbers that are merely cryptographically
121  * strong.  For many applications, however, this is acceptable.
122  *
123  * Exported interfaces ---- kernel output
124  * --------------------------------------
125  *
126  * The primary kernel interface is
127  *
128  * 	void get_random_bytes(void *buf, int nbytes);
129  *
130  * This interface will return the requested number of random bytes,
131  * and place it in the requested buffer.  This is equivalent to a
132  * read from /dev/urandom.
133  *
134  * For less critical applications, there are the functions:
135  *
136  * 	u32 get_random_u32()
137  * 	u64 get_random_u64()
138  * 	unsigned int get_random_int()
139  * 	unsigned long get_random_long()
140  *
141  * These are produced by a cryptographic RNG seeded from get_random_bytes,
142  * and so do not deplete the entropy pool as much.  These are recommended
143  * for most in-kernel operations *if the result is going to be stored in
144  * the kernel*.
145  *
146  * Specifically, the get_random_int() family do not attempt to do
147  * "anti-backtracking".  If you capture the state of the kernel (e.g.
148  * by snapshotting the VM), you can figure out previous get_random_int()
149  * return values.  But if the value is stored in the kernel anyway,
150  * this is not a problem.
151  *
152  * It *is* safe to expose get_random_int() output to attackers (e.g. as
153  * network cookies); given outputs 1..n, it's not feasible to predict
154  * outputs 0 or n+1.  The only concern is an attacker who breaks into
155  * the kernel later; the get_random_int() engine is not reseeded as
156  * often as the get_random_bytes() one.
157  *
158  * get_random_bytes() is needed for keys that need to stay secret after
159  * they are erased from the kernel.  For example, any key that will
160  * be wrapped and stored encrypted.  And session encryption keys: we'd
161  * like to know that after the session is closed and the keys erased,
162  * the plaintext is unrecoverable to someone who recorded the ciphertext.
163  *
164  * But for network ports/cookies, stack canaries, PRNG seeds, address
165  * space layout randomization, session *authentication* keys, or other
166  * applications where the sensitive data is stored in the kernel in
167  * plaintext for as long as it's sensitive, the get_random_int() family
168  * is just fine.
169  *
170  * Consider ASLR.  We want to keep the address space secret from an
171  * outside attacker while the process is running, but once the address
172  * space is torn down, it's of no use to an attacker any more.  And it's
173  * stored in kernel data structures as long as it's alive, so worrying
174  * about an attacker's ability to extrapolate it from the get_random_int()
175  * CRNG is silly.
176  *
177  * Even some cryptographic keys are safe to generate with get_random_int().
178  * In particular, keys for SipHash are generally fine.  Here, knowledge
179  * of the key authorizes you to do something to a kernel object (inject
180  * packets to a network connection, or flood a hash table), and the
181  * key is stored with the object being protected.  Once it goes away,
182  * we no longer care if anyone knows the key.
183  *
184  * prandom_u32()
185  * -------------
186  *
187  * For even weaker applications, see the pseudorandom generator
188  * prandom_u32(), prandom_max(), and prandom_bytes().  If the random
189  * numbers aren't security-critical at all, these are *far* cheaper.
190  * Useful for self-tests, random error simulation, randomized backoffs,
191  * and any other application where you trust that nobody is trying to
192  * maliciously mess with you by guessing the "random" numbers.
193  *
194  * Exported interfaces ---- input
195  * ==============================
196  *
197  * The current exported interfaces for gathering environmental noise
198  * from the devices are:
199  *
200  *	void add_device_randomness(const void *buf, unsigned int size);
201  * 	void add_input_randomness(unsigned int type, unsigned int code,
202  *                                unsigned int value);
203  *	void add_interrupt_randomness(int irq, int irq_flags);
204  * 	void add_disk_randomness(struct gendisk *disk);
205  *
206  * add_device_randomness() is for adding data to the random pool that
207  * is likely to differ between two devices (or possibly even per boot).
208  * This would be things like MAC addresses or serial numbers, or the
209  * read-out of the RTC. This does *not* add any actual entropy to the
210  * pool, but it initializes the pool to different values for devices
211  * that might otherwise be identical and have very little entropy
212  * available to them (particularly common in the embedded world).
213  *
214  * add_input_randomness() uses the input layer interrupt timing, as well as
215  * the event type information from the hardware.
216  *
217  * add_interrupt_randomness() uses the interrupt timing as random
218  * inputs to the entropy pool. Using the cycle counters and the irq source
219  * as inputs, it feeds the randomness roughly once a second.
220  *
221  * add_disk_randomness() uses what amounts to the seek time of block
222  * layer request events, on a per-disk_devt basis, as input to the
223  * entropy pool. Note that high-speed solid state drives with very low
224  * seek times do not make for good sources of entropy, as their seek
225  * times are usually fairly consistent.
226  *
227  * All of these routines try to estimate how many bits of randomness a
228  * particular randomness source.  They do this by keeping track of the
229  * first and second order deltas of the event timings.
230  *
231  * Ensuring unpredictability at system startup
232  * ============================================
233  *
234  * When any operating system starts up, it will go through a sequence
235  * of actions that are fairly predictable by an adversary, especially
236  * if the start-up does not involve interaction with a human operator.
237  * This reduces the actual number of bits of unpredictability in the
238  * entropy pool below the value in entropy_count.  In order to
239  * counteract this effect, it helps to carry information in the
240  * entropy pool across shut-downs and start-ups.  To do this, put the
241  * following lines an appropriate script which is run during the boot
242  * sequence:
243  *
244  *	echo "Initializing random number generator..."
245  *	random_seed=/var/run/random-seed
246  *	# Carry a random seed from start-up to start-up
247  *	# Load and then save the whole entropy pool
248  *	if [ -f $random_seed ]; then
249  *		cat $random_seed >/dev/urandom
250  *	else
251  *		touch $random_seed
252  *	fi
253  *	chmod 600 $random_seed
254  *	dd if=/dev/urandom of=$random_seed count=1 bs=512
255  *
256  * and the following lines in an appropriate script which is run as
257  * the system is shutdown:
258  *
259  *	# Carry a random seed from shut-down to start-up
260  *	# Save the whole entropy pool
261  *	echo "Saving random seed..."
262  *	random_seed=/var/run/random-seed
263  *	touch $random_seed
264  *	chmod 600 $random_seed
265  *	dd if=/dev/urandom of=$random_seed count=1 bs=512
266  *
267  * For example, on most modern systems using the System V init
268  * scripts, such code fragments would be found in
269  * /etc/rc.d/init.d/random.  On older Linux systems, the correct script
270  * location might be in /etc/rcb.d/rc.local or /etc/rc.d/rc.0.
271  *
272  * Effectively, these commands cause the contents of the entropy pool
273  * to be saved at shut-down time and reloaded into the entropy pool at
274  * start-up.  (The 'dd' in the addition to the bootup script is to
275  * make sure that /etc/random-seed is different for every start-up,
276  * even if the system crashes without executing rc.0.)  Even with
277  * complete knowledge of the start-up activities, predicting the state
278  * of the entropy pool requires knowledge of the previous history of
279  * the system.
280  *
281  * Configuring the /dev/random driver under Linux
282  * ==============================================
283  *
284  * The /dev/random driver under Linux uses minor numbers 8 and 9 of
285  * the /dev/mem major number (#1).  So if your system does not have
286  * /dev/random and /dev/urandom created already, they can be created
287  * by using the commands:
288  *
289  * 	mknod /dev/random c 1 8
290  * 	mknod /dev/urandom c 1 9
291  *
292  * Acknowledgements:
293  * =================
294  *
295  * Ideas for constructing this random number generator were derived
296  * from Pretty Good Privacy's random number generator, and from private
297  * discussions with Phil Karn.  Colin Plumb provided a faster random
298  * number generator, which speed up the mixing function of the entropy
299  * pool, taken from PGPfone.  Dale Worley has also contributed many
300  * useful ideas and suggestions to improve this driver.
301  *
302  * Any flaws in the design are solely my responsibility, and should
303  * not be attributed to the Phil, Colin, or any of authors of PGP.
304  *
305  * Further background information on this topic may be obtained from
306  * RFC 1750, "Randomness Recommendations for Security", by Donald
307  * Eastlake, Steve Crocker, and Jeff Schiller.
308  */
309 
310 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
311 
312 #include <linux/utsname.h>
313 #include <linux/module.h>
314 #include <linux/kernel.h>
315 #include <linux/major.h>
316 #include <linux/string.h>
317 #include <linux/fcntl.h>
318 #include <linux/slab.h>
319 #include <linux/random.h>
320 #include <linux/poll.h>
321 #include <linux/init.h>
322 #include <linux/fs.h>
323 #include <linux/genhd.h>
324 #include <linux/interrupt.h>
325 #include <linux/mm.h>
326 #include <linux/nodemask.h>
327 #include <linux/spinlock.h>
328 #include <linux/kthread.h>
329 #include <linux/percpu.h>
330 #include <linux/cryptohash.h>
331 #include <linux/fips.h>
332 #include <linux/ptrace.h>
333 #include <linux/workqueue.h>
334 #include <linux/irq.h>
335 #include <linux/ratelimit.h>
336 #include <linux/syscalls.h>
337 #include <linux/completion.h>
338 #include <linux/uuid.h>
339 #include <crypto/chacha.h>
340 
341 #include <asm/processor.h>
342 #include <linux/uaccess.h>
343 #include <asm/irq.h>
344 #include <asm/irq_regs.h>
345 #include <asm/io.h>
346 
347 #define CREATE_TRACE_POINTS
348 #include <trace/events/random.h>
349 
350 /* #define ADD_INTERRUPT_BENCH */
351 
352 /*
353  * Configuration information
354  */
355 #define INPUT_POOL_SHIFT	12
356 #define INPUT_POOL_WORDS	(1 << (INPUT_POOL_SHIFT-5))
357 #define OUTPUT_POOL_SHIFT	10
358 #define OUTPUT_POOL_WORDS	(1 << (OUTPUT_POOL_SHIFT-5))
359 #define EXTRACT_SIZE		10
360 
361 
362 #define LONGS(x) (((x) + sizeof(unsigned long) - 1)/sizeof(unsigned long))
363 
364 /*
365  * To allow fractional bits to be tracked, the entropy_count field is
366  * denominated in units of 1/8th bits.
367  *
368  * 2*(ENTROPY_SHIFT + poolbitshift) must <= 31, or the multiply in
369  * credit_entropy_bits() needs to be 64 bits wide.
370  */
371 #define ENTROPY_SHIFT 3
372 #define ENTROPY_BITS(r) ((r)->entropy_count >> ENTROPY_SHIFT)
373 
374 /*
375  * If the entropy count falls under this number of bits, then we
376  * should wake up processes which are selecting or polling on write
377  * access to /dev/random.
378  */
379 static int random_write_wakeup_bits = 28 * OUTPUT_POOL_WORDS;
380 
381 /*
382  * Originally, we used a primitive polynomial of degree .poolwords
383  * over GF(2).  The taps for various sizes are defined below.  They
384  * were chosen to be evenly spaced except for the last tap, which is 1
385  * to get the twisting happening as fast as possible.
386  *
387  * For the purposes of better mixing, we use the CRC-32 polynomial as
388  * well to make a (modified) twisted Generalized Feedback Shift
389  * Register.  (See M. Matsumoto & Y. Kurita, 1992.  Twisted GFSR
390  * generators.  ACM Transactions on Modeling and Computer Simulation
391  * 2(3):179-194.  Also see M. Matsumoto & Y. Kurita, 1994.  Twisted
392  * GFSR generators II.  ACM Transactions on Modeling and Computer
393  * Simulation 4:254-266)
394  *
395  * Thanks to Colin Plumb for suggesting this.
396  *
397  * The mixing operation is much less sensitive than the output hash,
398  * where we use SHA-1.  All that we want of mixing operation is that
399  * it be a good non-cryptographic hash; i.e. it not produce collisions
400  * when fed "random" data of the sort we expect to see.  As long as
401  * the pool state differs for different inputs, we have preserved the
402  * input entropy and done a good job.  The fact that an intelligent
403  * attacker can construct inputs that will produce controlled
404  * alterations to the pool's state is not important because we don't
405  * consider such inputs to contribute any randomness.  The only
406  * property we need with respect to them is that the attacker can't
407  * increase his/her knowledge of the pool's state.  Since all
408  * additions are reversible (knowing the final state and the input,
409  * you can reconstruct the initial state), if an attacker has any
410  * uncertainty about the initial state, he/she can only shuffle that
411  * uncertainty about, but never cause any collisions (which would
412  * decrease the uncertainty).
413  *
414  * Our mixing functions were analyzed by Lacharme, Roeck, Strubel, and
415  * Videau in their paper, "The Linux Pseudorandom Number Generator
416  * Revisited" (see: http://eprint.iacr.org/2012/251.pdf).  In their
417  * paper, they point out that we are not using a true Twisted GFSR,
418  * since Matsumoto & Kurita used a trinomial feedback polynomial (that
419  * is, with only three taps, instead of the six that we are using).
420  * As a result, the resulting polynomial is neither primitive nor
421  * irreducible, and hence does not have a maximal period over
422  * GF(2**32).  They suggest a slight change to the generator
423  * polynomial which improves the resulting TGFSR polynomial to be
424  * irreducible, which we have made here.
425  */
426 static const struct poolinfo {
427 	int poolbitshift, poolwords, poolbytes, poolfracbits;
428 #define S(x) ilog2(x)+5, (x), (x)*4, (x) << (ENTROPY_SHIFT+5)
429 	int tap1, tap2, tap3, tap4, tap5;
430 } poolinfo_table[] = {
431 	/* was: x^128 + x^103 + x^76 + x^51 +x^25 + x + 1 */
432 	/* x^128 + x^104 + x^76 + x^51 +x^25 + x + 1 */
433 	{ S(128),	104,	76,	51,	25,	1 },
434 };
435 
436 /*
437  * Static global variables
438  */
439 static DECLARE_WAIT_QUEUE_HEAD(random_write_wait);
440 static struct fasync_struct *fasync;
441 
442 static DEFINE_SPINLOCK(random_ready_list_lock);
443 static LIST_HEAD(random_ready_list);
444 
445 struct crng_state {
446 	__u32		state[16];
447 	unsigned long	init_time;
448 	spinlock_t	lock;
449 };
450 
451 static struct crng_state primary_crng = {
452 	.lock = __SPIN_LOCK_UNLOCKED(primary_crng.lock),
453 };
454 
455 /*
456  * crng_init =  0 --> Uninitialized
457  *		1 --> Initialized
458  *		2 --> Initialized from input_pool
459  *
460  * crng_init is protected by primary_crng->lock, and only increases
461  * its value (from 0->1->2).
462  */
463 static int crng_init = 0;
464 #define crng_ready() (likely(crng_init > 1))
465 static int crng_init_cnt = 0;
466 static unsigned long crng_global_init_time = 0;
467 #define CRNG_INIT_CNT_THRESH (2*CHACHA_KEY_SIZE)
468 static void _extract_crng(struct crng_state *crng, __u8 out[CHACHA_BLOCK_SIZE]);
469 static void _crng_backtrack_protect(struct crng_state *crng,
470 				    __u8 tmp[CHACHA_BLOCK_SIZE], int used);
471 static void process_random_ready_list(void);
472 static void _get_random_bytes(void *buf, int nbytes);
473 
474 static struct ratelimit_state unseeded_warning =
475 	RATELIMIT_STATE_INIT("warn_unseeded_randomness", HZ, 3);
476 static struct ratelimit_state urandom_warning =
477 	RATELIMIT_STATE_INIT("warn_urandom_randomness", HZ, 3);
478 
479 static int ratelimit_disable __read_mostly;
480 
481 module_param_named(ratelimit_disable, ratelimit_disable, int, 0644);
482 MODULE_PARM_DESC(ratelimit_disable, "Disable random ratelimit suppression");
483 
484 /**********************************************************************
485  *
486  * OS independent entropy store.   Here are the functions which handle
487  * storing entropy in an entropy pool.
488  *
489  **********************************************************************/
490 
491 struct entropy_store;
492 struct entropy_store {
493 	/* read-only data: */
494 	const struct poolinfo *poolinfo;
495 	__u32 *pool;
496 	const char *name;
497 
498 	/* read-write data: */
499 	spinlock_t lock;
500 	unsigned short add_ptr;
501 	unsigned short input_rotate;
502 	int entropy_count;
503 	unsigned int initialized:1;
504 	unsigned int last_data_init:1;
505 	__u8 last_data[EXTRACT_SIZE];
506 };
507 
508 static ssize_t extract_entropy(struct entropy_store *r, void *buf,
509 			       size_t nbytes, int min, int rsvd);
510 static ssize_t _extract_entropy(struct entropy_store *r, void *buf,
511 				size_t nbytes, int fips);
512 
513 static void crng_reseed(struct crng_state *crng, struct entropy_store *r);
514 static __u32 input_pool_data[INPUT_POOL_WORDS] __latent_entropy;
515 
516 static struct entropy_store input_pool = {
517 	.poolinfo = &poolinfo_table[0],
518 	.name = "input",
519 	.lock = __SPIN_LOCK_UNLOCKED(input_pool.lock),
520 	.pool = input_pool_data
521 };
522 
523 static __u32 const twist_table[8] = {
524 	0x00000000, 0x3b6e20c8, 0x76dc4190, 0x4db26158,
525 	0xedb88320, 0xd6d6a3e8, 0x9b64c2b0, 0xa00ae278 };
526 
527 /*
528  * This function adds bytes into the entropy "pool".  It does not
529  * update the entropy estimate.  The caller should call
530  * credit_entropy_bits if this is appropriate.
531  *
532  * The pool is stirred with a primitive polynomial of the appropriate
533  * degree, and then twisted.  We twist by three bits at a time because
534  * it's cheap to do so and helps slightly in the expected case where
535  * the entropy is concentrated in the low-order bits.
536  */
537 static void _mix_pool_bytes(struct entropy_store *r, const void *in,
538 			    int nbytes)
539 {
540 	unsigned long i, tap1, tap2, tap3, tap4, tap5;
541 	int input_rotate;
542 	int wordmask = r->poolinfo->poolwords - 1;
543 	const char *bytes = in;
544 	__u32 w;
545 
546 	tap1 = r->poolinfo->tap1;
547 	tap2 = r->poolinfo->tap2;
548 	tap3 = r->poolinfo->tap3;
549 	tap4 = r->poolinfo->tap4;
550 	tap5 = r->poolinfo->tap5;
551 
552 	input_rotate = r->input_rotate;
553 	i = r->add_ptr;
554 
555 	/* mix one byte at a time to simplify size handling and churn faster */
556 	while (nbytes--) {
557 		w = rol32(*bytes++, input_rotate);
558 		i = (i - 1) & wordmask;
559 
560 		/* XOR in the various taps */
561 		w ^= r->pool[i];
562 		w ^= r->pool[(i + tap1) & wordmask];
563 		w ^= r->pool[(i + tap2) & wordmask];
564 		w ^= r->pool[(i + tap3) & wordmask];
565 		w ^= r->pool[(i + tap4) & wordmask];
566 		w ^= r->pool[(i + tap5) & wordmask];
567 
568 		/* Mix the result back in with a twist */
569 		r->pool[i] = (w >> 3) ^ twist_table[w & 7];
570 
571 		/*
572 		 * Normally, we add 7 bits of rotation to the pool.
573 		 * At the beginning of the pool, add an extra 7 bits
574 		 * rotation, so that successive passes spread the
575 		 * input bits across the pool evenly.
576 		 */
577 		input_rotate = (input_rotate + (i ? 7 : 14)) & 31;
578 	}
579 
580 	r->input_rotate = input_rotate;
581 	r->add_ptr = i;
582 }
583 
584 static void __mix_pool_bytes(struct entropy_store *r, const void *in,
585 			     int nbytes)
586 {
587 	trace_mix_pool_bytes_nolock(r->name, nbytes, _RET_IP_);
588 	_mix_pool_bytes(r, in, nbytes);
589 }
590 
591 static void mix_pool_bytes(struct entropy_store *r, const void *in,
592 			   int nbytes)
593 {
594 	unsigned long flags;
595 
596 	trace_mix_pool_bytes(r->name, nbytes, _RET_IP_);
597 	spin_lock_irqsave(&r->lock, flags);
598 	_mix_pool_bytes(r, in, nbytes);
599 	spin_unlock_irqrestore(&r->lock, flags);
600 }
601 
602 struct fast_pool {
603 	__u32		pool[4];
604 	unsigned long	last;
605 	unsigned short	reg_idx;
606 	unsigned char	count;
607 };
608 
609 /*
610  * This is a fast mixing routine used by the interrupt randomness
611  * collector.  It's hardcoded for an 128 bit pool and assumes that any
612  * locks that might be needed are taken by the caller.
613  */
614 static void fast_mix(struct fast_pool *f)
615 {
616 	__u32 a = f->pool[0],	b = f->pool[1];
617 	__u32 c = f->pool[2],	d = f->pool[3];
618 
619 	a += b;			c += d;
620 	b = rol32(b, 6);	d = rol32(d, 27);
621 	d ^= a;			b ^= c;
622 
623 	a += b;			c += d;
624 	b = rol32(b, 16);	d = rol32(d, 14);
625 	d ^= a;			b ^= c;
626 
627 	a += b;			c += d;
628 	b = rol32(b, 6);	d = rol32(d, 27);
629 	d ^= a;			b ^= c;
630 
631 	a += b;			c += d;
632 	b = rol32(b, 16);	d = rol32(d, 14);
633 	d ^= a;			b ^= c;
634 
635 	f->pool[0] = a;  f->pool[1] = b;
636 	f->pool[2] = c;  f->pool[3] = d;
637 	f->count++;
638 }
639 
640 static void process_random_ready_list(void)
641 {
642 	unsigned long flags;
643 	struct random_ready_callback *rdy, *tmp;
644 
645 	spin_lock_irqsave(&random_ready_list_lock, flags);
646 	list_for_each_entry_safe(rdy, tmp, &random_ready_list, list) {
647 		struct module *owner = rdy->owner;
648 
649 		list_del_init(&rdy->list);
650 		rdy->func(rdy);
651 		module_put(owner);
652 	}
653 	spin_unlock_irqrestore(&random_ready_list_lock, flags);
654 }
655 
656 /*
657  * Credit (or debit) the entropy store with n bits of entropy.
658  * Use credit_entropy_bits_safe() if the value comes from userspace
659  * or otherwise should be checked for extreme values.
660  */
661 static void credit_entropy_bits(struct entropy_store *r, int nbits)
662 {
663 	int entropy_count, orig, has_initialized = 0;
664 	const int pool_size = r->poolinfo->poolfracbits;
665 	int nfrac = nbits << ENTROPY_SHIFT;
666 
667 	if (!nbits)
668 		return;
669 
670 retry:
671 	entropy_count = orig = READ_ONCE(r->entropy_count);
672 	if (nfrac < 0) {
673 		/* Debit */
674 		entropy_count += nfrac;
675 	} else {
676 		/*
677 		 * Credit: we have to account for the possibility of
678 		 * overwriting already present entropy.	 Even in the
679 		 * ideal case of pure Shannon entropy, new contributions
680 		 * approach the full value asymptotically:
681 		 *
682 		 * entropy <- entropy + (pool_size - entropy) *
683 		 *	(1 - exp(-add_entropy/pool_size))
684 		 *
685 		 * For add_entropy <= pool_size/2 then
686 		 * (1 - exp(-add_entropy/pool_size)) >=
687 		 *    (add_entropy/pool_size)*0.7869...
688 		 * so we can approximate the exponential with
689 		 * 3/4*add_entropy/pool_size and still be on the
690 		 * safe side by adding at most pool_size/2 at a time.
691 		 *
692 		 * The use of pool_size-2 in the while statement is to
693 		 * prevent rounding artifacts from making the loop
694 		 * arbitrarily long; this limits the loop to log2(pool_size)*2
695 		 * turns no matter how large nbits is.
696 		 */
697 		int pnfrac = nfrac;
698 		const int s = r->poolinfo->poolbitshift + ENTROPY_SHIFT + 2;
699 		/* The +2 corresponds to the /4 in the denominator */
700 
701 		do {
702 			unsigned int anfrac = min(pnfrac, pool_size/2);
703 			unsigned int add =
704 				((pool_size - entropy_count)*anfrac*3) >> s;
705 
706 			entropy_count += add;
707 			pnfrac -= anfrac;
708 		} while (unlikely(entropy_count < pool_size-2 && pnfrac));
709 	}
710 
711 	if (WARN_ON(entropy_count < 0)) {
712 		pr_warn("negative entropy/overflow: pool %s count %d\n",
713 			r->name, entropy_count);
714 		entropy_count = 0;
715 	} else if (entropy_count > pool_size)
716 		entropy_count = pool_size;
717 	if (cmpxchg(&r->entropy_count, orig, entropy_count) != orig)
718 		goto retry;
719 
720 	if (has_initialized) {
721 		r->initialized = 1;
722 		kill_fasync(&fasync, SIGIO, POLL_IN);
723 	}
724 
725 	trace_credit_entropy_bits(r->name, nbits,
726 				  entropy_count >> ENTROPY_SHIFT, _RET_IP_);
727 
728 	if (r == &input_pool) {
729 		int entropy_bits = entropy_count >> ENTROPY_SHIFT;
730 
731 		if (crng_init < 2) {
732 			if (entropy_bits < 128)
733 				return;
734 			crng_reseed(&primary_crng, r);
735 			entropy_bits = ENTROPY_BITS(r);
736 		}
737 	}
738 }
739 
740 static int credit_entropy_bits_safe(struct entropy_store *r, int nbits)
741 {
742 	const int nbits_max = r->poolinfo->poolwords * 32;
743 
744 	if (nbits < 0)
745 		return -EINVAL;
746 
747 	/* Cap the value to avoid overflows */
748 	nbits = min(nbits,  nbits_max);
749 
750 	credit_entropy_bits(r, nbits);
751 	return 0;
752 }
753 
754 /*********************************************************************
755  *
756  * CRNG using CHACHA20
757  *
758  *********************************************************************/
759 
760 #define CRNG_RESEED_INTERVAL (300*HZ)
761 
762 static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait);
763 
764 #ifdef CONFIG_NUMA
765 /*
766  * Hack to deal with crazy userspace progams when they are all trying
767  * to access /dev/urandom in parallel.  The programs are almost
768  * certainly doing something terribly wrong, but we'll work around
769  * their brain damage.
770  */
771 static struct crng_state **crng_node_pool __read_mostly;
772 #endif
773 
774 static void invalidate_batched_entropy(void);
775 static void numa_crng_init(void);
776 
777 static bool trust_cpu __ro_after_init = IS_ENABLED(CONFIG_RANDOM_TRUST_CPU);
778 static int __init parse_trust_cpu(char *arg)
779 {
780 	return kstrtobool(arg, &trust_cpu);
781 }
782 early_param("random.trust_cpu", parse_trust_cpu);
783 
784 static void crng_initialize(struct crng_state *crng)
785 {
786 	int		i;
787 	int		arch_init = 1;
788 	unsigned long	rv;
789 
790 	memcpy(&crng->state[0], "expand 32-byte k", 16);
791 	if (crng == &primary_crng)
792 		_extract_entropy(&input_pool, &crng->state[4],
793 				 sizeof(__u32) * 12, 0);
794 	else
795 		_get_random_bytes(&crng->state[4], sizeof(__u32) * 12);
796 	for (i = 4; i < 16; i++) {
797 		if (!arch_get_random_seed_long(&rv) &&
798 		    !arch_get_random_long(&rv)) {
799 			rv = random_get_entropy();
800 			arch_init = 0;
801 		}
802 		crng->state[i] ^= rv;
803 	}
804 	if (trust_cpu && arch_init && crng == &primary_crng) {
805 		invalidate_batched_entropy();
806 		numa_crng_init();
807 		crng_init = 2;
808 		pr_notice("crng done (trusting CPU's manufacturer)\n");
809 	}
810 	crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1;
811 }
812 
813 #ifdef CONFIG_NUMA
814 static void do_numa_crng_init(struct work_struct *work)
815 {
816 	int i;
817 	struct crng_state *crng;
818 	struct crng_state **pool;
819 
820 	pool = kcalloc(nr_node_ids, sizeof(*pool), GFP_KERNEL|__GFP_NOFAIL);
821 	for_each_online_node(i) {
822 		crng = kmalloc_node(sizeof(struct crng_state),
823 				    GFP_KERNEL | __GFP_NOFAIL, i);
824 		spin_lock_init(&crng->lock);
825 		crng_initialize(crng);
826 		pool[i] = crng;
827 	}
828 	mb();
829 	if (cmpxchg(&crng_node_pool, NULL, pool)) {
830 		for_each_node(i)
831 			kfree(pool[i]);
832 		kfree(pool);
833 	}
834 }
835 
836 static DECLARE_WORK(numa_crng_init_work, do_numa_crng_init);
837 
838 static void numa_crng_init(void)
839 {
840 	schedule_work(&numa_crng_init_work);
841 }
842 #else
843 static void numa_crng_init(void) {}
844 #endif
845 
846 /*
847  * crng_fast_load() can be called by code in the interrupt service
848  * path.  So we can't afford to dilly-dally.
849  */
850 static int crng_fast_load(const char *cp, size_t len)
851 {
852 	unsigned long flags;
853 	char *p;
854 
855 	if (!spin_trylock_irqsave(&primary_crng.lock, flags))
856 		return 0;
857 	if (crng_init != 0) {
858 		spin_unlock_irqrestore(&primary_crng.lock, flags);
859 		return 0;
860 	}
861 	p = (unsigned char *) &primary_crng.state[4];
862 	while (len > 0 && crng_init_cnt < CRNG_INIT_CNT_THRESH) {
863 		p[crng_init_cnt % CHACHA_KEY_SIZE] ^= *cp;
864 		cp++; crng_init_cnt++; len--;
865 	}
866 	spin_unlock_irqrestore(&primary_crng.lock, flags);
867 	if (crng_init_cnt >= CRNG_INIT_CNT_THRESH) {
868 		invalidate_batched_entropy();
869 		crng_init = 1;
870 		pr_notice("fast init done\n");
871 	}
872 	return 1;
873 }
874 
875 /*
876  * crng_slow_load() is called by add_device_randomness, which has two
877  * attributes.  (1) We can't trust the buffer passed to it is
878  * guaranteed to be unpredictable (so it might not have any entropy at
879  * all), and (2) it doesn't have the performance constraints of
880  * crng_fast_load().
881  *
882  * So we do something more comprehensive which is guaranteed to touch
883  * all of the primary_crng's state, and which uses a LFSR with a
884  * period of 255 as part of the mixing algorithm.  Finally, we do
885  * *not* advance crng_init_cnt since buffer we may get may be something
886  * like a fixed DMI table (for example), which might very well be
887  * unique to the machine, but is otherwise unvarying.
888  */
889 static int crng_slow_load(const char *cp, size_t len)
890 {
891 	unsigned long		flags;
892 	static unsigned char	lfsr = 1;
893 	unsigned char		tmp;
894 	unsigned		i, max = CHACHA_KEY_SIZE;
895 	const char *		src_buf = cp;
896 	char *			dest_buf = (char *) &primary_crng.state[4];
897 
898 	if (!spin_trylock_irqsave(&primary_crng.lock, flags))
899 		return 0;
900 	if (crng_init != 0) {
901 		spin_unlock_irqrestore(&primary_crng.lock, flags);
902 		return 0;
903 	}
904 	if (len > max)
905 		max = len;
906 
907 	for (i = 0; i < max ; i++) {
908 		tmp = lfsr;
909 		lfsr >>= 1;
910 		if (tmp & 1)
911 			lfsr ^= 0xE1;
912 		tmp = dest_buf[i % CHACHA_KEY_SIZE];
913 		dest_buf[i % CHACHA_KEY_SIZE] ^= src_buf[i % len] ^ lfsr;
914 		lfsr += (tmp << 3) | (tmp >> 5);
915 	}
916 	spin_unlock_irqrestore(&primary_crng.lock, flags);
917 	return 1;
918 }
919 
920 static void crng_reseed(struct crng_state *crng, struct entropy_store *r)
921 {
922 	unsigned long	flags;
923 	int		i, num;
924 	union {
925 		__u8	block[CHACHA_BLOCK_SIZE];
926 		__u32	key[8];
927 	} buf;
928 
929 	if (r) {
930 		num = extract_entropy(r, &buf, 32, 16, 0);
931 		if (num == 0)
932 			return;
933 	} else {
934 		_extract_crng(&primary_crng, buf.block);
935 		_crng_backtrack_protect(&primary_crng, buf.block,
936 					CHACHA_KEY_SIZE);
937 	}
938 	spin_lock_irqsave(&crng->lock, flags);
939 	for (i = 0; i < 8; i++) {
940 		unsigned long	rv;
941 		if (!arch_get_random_seed_long(&rv) &&
942 		    !arch_get_random_long(&rv))
943 			rv = random_get_entropy();
944 		crng->state[i+4] ^= buf.key[i] ^ rv;
945 	}
946 	memzero_explicit(&buf, sizeof(buf));
947 	crng->init_time = jiffies;
948 	spin_unlock_irqrestore(&crng->lock, flags);
949 	if (crng == &primary_crng && crng_init < 2) {
950 		invalidate_batched_entropy();
951 		numa_crng_init();
952 		crng_init = 2;
953 		process_random_ready_list();
954 		wake_up_interruptible(&crng_init_wait);
955 		kill_fasync(&fasync, SIGIO, POLL_IN);
956 		pr_notice("crng init done\n");
957 		if (unseeded_warning.missed) {
958 			pr_notice("%d get_random_xx warning(s) missed due to ratelimiting\n",
959 				  unseeded_warning.missed);
960 			unseeded_warning.missed = 0;
961 		}
962 		if (urandom_warning.missed) {
963 			pr_notice("%d urandom warning(s) missed due to ratelimiting\n",
964 				  urandom_warning.missed);
965 			urandom_warning.missed = 0;
966 		}
967 	}
968 }
969 
970 static void _extract_crng(struct crng_state *crng,
971 			  __u8 out[CHACHA_BLOCK_SIZE])
972 {
973 	unsigned long v, flags;
974 
975 	if (crng_ready() &&
976 	    (time_after(crng_global_init_time, crng->init_time) ||
977 	     time_after(jiffies, crng->init_time + CRNG_RESEED_INTERVAL)))
978 		crng_reseed(crng, crng == &primary_crng ? &input_pool : NULL);
979 	spin_lock_irqsave(&crng->lock, flags);
980 	if (arch_get_random_long(&v))
981 		crng->state[14] ^= v;
982 	chacha20_block(&crng->state[0], out);
983 	if (crng->state[12] == 0)
984 		crng->state[13]++;
985 	spin_unlock_irqrestore(&crng->lock, flags);
986 }
987 
988 static void extract_crng(__u8 out[CHACHA_BLOCK_SIZE])
989 {
990 	struct crng_state *crng = NULL;
991 
992 #ifdef CONFIG_NUMA
993 	if (crng_node_pool)
994 		crng = crng_node_pool[numa_node_id()];
995 	if (crng == NULL)
996 #endif
997 		crng = &primary_crng;
998 	_extract_crng(crng, out);
999 }
1000 
1001 /*
1002  * Use the leftover bytes from the CRNG block output (if there is
1003  * enough) to mutate the CRNG key to provide backtracking protection.
1004  */
1005 static void _crng_backtrack_protect(struct crng_state *crng,
1006 				    __u8 tmp[CHACHA_BLOCK_SIZE], int used)
1007 {
1008 	unsigned long	flags;
1009 	__u32		*s, *d;
1010 	int		i;
1011 
1012 	used = round_up(used, sizeof(__u32));
1013 	if (used + CHACHA_KEY_SIZE > CHACHA_BLOCK_SIZE) {
1014 		extract_crng(tmp);
1015 		used = 0;
1016 	}
1017 	spin_lock_irqsave(&crng->lock, flags);
1018 	s = (__u32 *) &tmp[used];
1019 	d = &crng->state[4];
1020 	for (i=0; i < 8; i++)
1021 		*d++ ^= *s++;
1022 	spin_unlock_irqrestore(&crng->lock, flags);
1023 }
1024 
1025 static void crng_backtrack_protect(__u8 tmp[CHACHA_BLOCK_SIZE], int used)
1026 {
1027 	struct crng_state *crng = NULL;
1028 
1029 #ifdef CONFIG_NUMA
1030 	if (crng_node_pool)
1031 		crng = crng_node_pool[numa_node_id()];
1032 	if (crng == NULL)
1033 #endif
1034 		crng = &primary_crng;
1035 	_crng_backtrack_protect(crng, tmp, used);
1036 }
1037 
1038 static ssize_t extract_crng_user(void __user *buf, size_t nbytes)
1039 {
1040 	ssize_t ret = 0, i = CHACHA_BLOCK_SIZE;
1041 	__u8 tmp[CHACHA_BLOCK_SIZE] __aligned(4);
1042 	int large_request = (nbytes > 256);
1043 
1044 	while (nbytes) {
1045 		if (large_request && need_resched()) {
1046 			if (signal_pending(current)) {
1047 				if (ret == 0)
1048 					ret = -ERESTARTSYS;
1049 				break;
1050 			}
1051 			schedule();
1052 		}
1053 
1054 		extract_crng(tmp);
1055 		i = min_t(int, nbytes, CHACHA_BLOCK_SIZE);
1056 		if (copy_to_user(buf, tmp, i)) {
1057 			ret = -EFAULT;
1058 			break;
1059 		}
1060 
1061 		nbytes -= i;
1062 		buf += i;
1063 		ret += i;
1064 	}
1065 	crng_backtrack_protect(tmp, i);
1066 
1067 	/* Wipe data just written to memory */
1068 	memzero_explicit(tmp, sizeof(tmp));
1069 
1070 	return ret;
1071 }
1072 
1073 
1074 /*********************************************************************
1075  *
1076  * Entropy input management
1077  *
1078  *********************************************************************/
1079 
1080 /* There is one of these per entropy source */
1081 struct timer_rand_state {
1082 	cycles_t last_time;
1083 	long last_delta, last_delta2;
1084 };
1085 
1086 #define INIT_TIMER_RAND_STATE { INITIAL_JIFFIES, };
1087 
1088 /*
1089  * Add device- or boot-specific data to the input pool to help
1090  * initialize it.
1091  *
1092  * None of this adds any entropy; it is meant to avoid the problem of
1093  * the entropy pool having similar initial state across largely
1094  * identical devices.
1095  */
1096 void add_device_randomness(const void *buf, unsigned int size)
1097 {
1098 	unsigned long time = random_get_entropy() ^ jiffies;
1099 	unsigned long flags;
1100 
1101 	if (!crng_ready() && size)
1102 		crng_slow_load(buf, size);
1103 
1104 	trace_add_device_randomness(size, _RET_IP_);
1105 	spin_lock_irqsave(&input_pool.lock, flags);
1106 	_mix_pool_bytes(&input_pool, buf, size);
1107 	_mix_pool_bytes(&input_pool, &time, sizeof(time));
1108 	spin_unlock_irqrestore(&input_pool.lock, flags);
1109 }
1110 EXPORT_SYMBOL(add_device_randomness);
1111 
1112 static struct timer_rand_state input_timer_state = INIT_TIMER_RAND_STATE;
1113 
1114 /*
1115  * This function adds entropy to the entropy "pool" by using timing
1116  * delays.  It uses the timer_rand_state structure to make an estimate
1117  * of how many bits of entropy this call has added to the pool.
1118  *
1119  * The number "num" is also added to the pool - it should somehow describe
1120  * the type of event which just happened.  This is currently 0-255 for
1121  * keyboard scan codes, and 256 upwards for interrupts.
1122  *
1123  */
1124 static void add_timer_randomness(struct timer_rand_state *state, unsigned num)
1125 {
1126 	struct entropy_store	*r;
1127 	struct {
1128 		long jiffies;
1129 		unsigned cycles;
1130 		unsigned num;
1131 	} sample;
1132 	long delta, delta2, delta3;
1133 
1134 	sample.jiffies = jiffies;
1135 	sample.cycles = random_get_entropy();
1136 	sample.num = num;
1137 	r = &input_pool;
1138 	mix_pool_bytes(r, &sample, sizeof(sample));
1139 
1140 	/*
1141 	 * Calculate number of bits of randomness we probably added.
1142 	 * We take into account the first, second and third-order deltas
1143 	 * in order to make our estimate.
1144 	 */
1145 	delta = sample.jiffies - state->last_time;
1146 	state->last_time = sample.jiffies;
1147 
1148 	delta2 = delta - state->last_delta;
1149 	state->last_delta = delta;
1150 
1151 	delta3 = delta2 - state->last_delta2;
1152 	state->last_delta2 = delta2;
1153 
1154 	if (delta < 0)
1155 		delta = -delta;
1156 	if (delta2 < 0)
1157 		delta2 = -delta2;
1158 	if (delta3 < 0)
1159 		delta3 = -delta3;
1160 	if (delta > delta2)
1161 		delta = delta2;
1162 	if (delta > delta3)
1163 		delta = delta3;
1164 
1165 	/*
1166 	 * delta is now minimum absolute delta.
1167 	 * Round down by 1 bit on general principles,
1168 	 * and limit entropy estimate to 12 bits.
1169 	 */
1170 	credit_entropy_bits(r, min_t(int, fls(delta>>1), 11));
1171 }
1172 
1173 void add_input_randomness(unsigned int type, unsigned int code,
1174 				 unsigned int value)
1175 {
1176 	static unsigned char last_value;
1177 
1178 	/* ignore autorepeat and the like */
1179 	if (value == last_value)
1180 		return;
1181 
1182 	last_value = value;
1183 	add_timer_randomness(&input_timer_state,
1184 			     (type << 4) ^ code ^ (code >> 4) ^ value);
1185 	trace_add_input_randomness(ENTROPY_BITS(&input_pool));
1186 }
1187 EXPORT_SYMBOL_GPL(add_input_randomness);
1188 
1189 static DEFINE_PER_CPU(struct fast_pool, irq_randomness);
1190 
1191 #ifdef ADD_INTERRUPT_BENCH
1192 static unsigned long avg_cycles, avg_deviation;
1193 
1194 #define AVG_SHIFT 8     /* Exponential average factor k=1/256 */
1195 #define FIXED_1_2 (1 << (AVG_SHIFT-1))
1196 
1197 static void add_interrupt_bench(cycles_t start)
1198 {
1199         long delta = random_get_entropy() - start;
1200 
1201         /* Use a weighted moving average */
1202         delta = delta - ((avg_cycles + FIXED_1_2) >> AVG_SHIFT);
1203         avg_cycles += delta;
1204         /* And average deviation */
1205         delta = abs(delta) - ((avg_deviation + FIXED_1_2) >> AVG_SHIFT);
1206         avg_deviation += delta;
1207 }
1208 #else
1209 #define add_interrupt_bench(x)
1210 #endif
1211 
1212 static __u32 get_reg(struct fast_pool *f, struct pt_regs *regs)
1213 {
1214 	__u32 *ptr = (__u32 *) regs;
1215 	unsigned int idx;
1216 
1217 	if (regs == NULL)
1218 		return 0;
1219 	idx = READ_ONCE(f->reg_idx);
1220 	if (idx >= sizeof(struct pt_regs) / sizeof(__u32))
1221 		idx = 0;
1222 	ptr += idx++;
1223 	WRITE_ONCE(f->reg_idx, idx);
1224 	return *ptr;
1225 }
1226 
1227 void add_interrupt_randomness(int irq, int irq_flags)
1228 {
1229 	struct entropy_store	*r;
1230 	struct fast_pool	*fast_pool = this_cpu_ptr(&irq_randomness);
1231 	struct pt_regs		*regs = get_irq_regs();
1232 	unsigned long		now = jiffies;
1233 	cycles_t		cycles = random_get_entropy();
1234 	__u32			c_high, j_high;
1235 	__u64			ip;
1236 	unsigned long		seed;
1237 	int			credit = 0;
1238 
1239 	if (cycles == 0)
1240 		cycles = get_reg(fast_pool, regs);
1241 	c_high = (sizeof(cycles) > 4) ? cycles >> 32 : 0;
1242 	j_high = (sizeof(now) > 4) ? now >> 32 : 0;
1243 	fast_pool->pool[0] ^= cycles ^ j_high ^ irq;
1244 	fast_pool->pool[1] ^= now ^ c_high;
1245 	ip = regs ? instruction_pointer(regs) : _RET_IP_;
1246 	fast_pool->pool[2] ^= ip;
1247 	fast_pool->pool[3] ^= (sizeof(ip) > 4) ? ip >> 32 :
1248 		get_reg(fast_pool, regs);
1249 
1250 	fast_mix(fast_pool);
1251 	add_interrupt_bench(cycles);
1252 
1253 	if (unlikely(crng_init == 0)) {
1254 		if ((fast_pool->count >= 64) &&
1255 		    crng_fast_load((char *) fast_pool->pool,
1256 				   sizeof(fast_pool->pool))) {
1257 			fast_pool->count = 0;
1258 			fast_pool->last = now;
1259 		}
1260 		return;
1261 	}
1262 
1263 	if ((fast_pool->count < 64) &&
1264 	    !time_after(now, fast_pool->last + HZ))
1265 		return;
1266 
1267 	r = &input_pool;
1268 	if (!spin_trylock(&r->lock))
1269 		return;
1270 
1271 	fast_pool->last = now;
1272 	__mix_pool_bytes(r, &fast_pool->pool, sizeof(fast_pool->pool));
1273 
1274 	/*
1275 	 * If we have architectural seed generator, produce a seed and
1276 	 * add it to the pool.  For the sake of paranoia don't let the
1277 	 * architectural seed generator dominate the input from the
1278 	 * interrupt noise.
1279 	 */
1280 	if (arch_get_random_seed_long(&seed)) {
1281 		__mix_pool_bytes(r, &seed, sizeof(seed));
1282 		credit = 1;
1283 	}
1284 	spin_unlock(&r->lock);
1285 
1286 	fast_pool->count = 0;
1287 
1288 	/* award one bit for the contents of the fast pool */
1289 	credit_entropy_bits(r, credit + 1);
1290 }
1291 EXPORT_SYMBOL_GPL(add_interrupt_randomness);
1292 
1293 #ifdef CONFIG_BLOCK
1294 void add_disk_randomness(struct gendisk *disk)
1295 {
1296 	if (!disk || !disk->random)
1297 		return;
1298 	/* first major is 1, so we get >= 0x200 here */
1299 	add_timer_randomness(disk->random, 0x100 + disk_devt(disk));
1300 	trace_add_disk_randomness(disk_devt(disk), ENTROPY_BITS(&input_pool));
1301 }
1302 EXPORT_SYMBOL_GPL(add_disk_randomness);
1303 #endif
1304 
1305 /*********************************************************************
1306  *
1307  * Entropy extraction routines
1308  *
1309  *********************************************************************/
1310 
1311 /*
1312  * This function decides how many bytes to actually take from the
1313  * given pool, and also debits the entropy count accordingly.
1314  */
1315 static size_t account(struct entropy_store *r, size_t nbytes, int min,
1316 		      int reserved)
1317 {
1318 	int entropy_count, orig, have_bytes;
1319 	size_t ibytes, nfrac;
1320 
1321 	BUG_ON(r->entropy_count > r->poolinfo->poolfracbits);
1322 
1323 	/* Can we pull enough? */
1324 retry:
1325 	entropy_count = orig = READ_ONCE(r->entropy_count);
1326 	ibytes = nbytes;
1327 	/* never pull more than available */
1328 	have_bytes = entropy_count >> (ENTROPY_SHIFT + 3);
1329 
1330 	if ((have_bytes -= reserved) < 0)
1331 		have_bytes = 0;
1332 	ibytes = min_t(size_t, ibytes, have_bytes);
1333 	if (ibytes < min)
1334 		ibytes = 0;
1335 
1336 	if (WARN_ON(entropy_count < 0)) {
1337 		pr_warn("negative entropy count: pool %s count %d\n",
1338 			r->name, entropy_count);
1339 		entropy_count = 0;
1340 	}
1341 	nfrac = ibytes << (ENTROPY_SHIFT + 3);
1342 	if ((size_t) entropy_count > nfrac)
1343 		entropy_count -= nfrac;
1344 	else
1345 		entropy_count = 0;
1346 
1347 	if (cmpxchg(&r->entropy_count, orig, entropy_count) != orig)
1348 		goto retry;
1349 
1350 	trace_debit_entropy(r->name, 8 * ibytes);
1351 	if (ibytes && ENTROPY_BITS(r) < random_write_wakeup_bits) {
1352 		wake_up_interruptible(&random_write_wait);
1353 		kill_fasync(&fasync, SIGIO, POLL_OUT);
1354 	}
1355 
1356 	return ibytes;
1357 }
1358 
1359 /*
1360  * This function does the actual extraction for extract_entropy and
1361  * extract_entropy_user.
1362  *
1363  * Note: we assume that .poolwords is a multiple of 16 words.
1364  */
1365 static void extract_buf(struct entropy_store *r, __u8 *out)
1366 {
1367 	int i;
1368 	union {
1369 		__u32 w[5];
1370 		unsigned long l[LONGS(20)];
1371 	} hash;
1372 	__u32 workspace[SHA_WORKSPACE_WORDS];
1373 	unsigned long flags;
1374 
1375 	/*
1376 	 * If we have an architectural hardware random number
1377 	 * generator, use it for SHA's initial vector
1378 	 */
1379 	sha_init(hash.w);
1380 	for (i = 0; i < LONGS(20); i++) {
1381 		unsigned long v;
1382 		if (!arch_get_random_long(&v))
1383 			break;
1384 		hash.l[i] = v;
1385 	}
1386 
1387 	/* Generate a hash across the pool, 16 words (512 bits) at a time */
1388 	spin_lock_irqsave(&r->lock, flags);
1389 	for (i = 0; i < r->poolinfo->poolwords; i += 16)
1390 		sha_transform(hash.w, (__u8 *)(r->pool + i), workspace);
1391 
1392 	/*
1393 	 * We mix the hash back into the pool to prevent backtracking
1394 	 * attacks (where the attacker knows the state of the pool
1395 	 * plus the current outputs, and attempts to find previous
1396 	 * ouputs), unless the hash function can be inverted. By
1397 	 * mixing at least a SHA1 worth of hash data back, we make
1398 	 * brute-forcing the feedback as hard as brute-forcing the
1399 	 * hash.
1400 	 */
1401 	__mix_pool_bytes(r, hash.w, sizeof(hash.w));
1402 	spin_unlock_irqrestore(&r->lock, flags);
1403 
1404 	memzero_explicit(workspace, sizeof(workspace));
1405 
1406 	/*
1407 	 * In case the hash function has some recognizable output
1408 	 * pattern, we fold it in half. Thus, we always feed back
1409 	 * twice as much data as we output.
1410 	 */
1411 	hash.w[0] ^= hash.w[3];
1412 	hash.w[1] ^= hash.w[4];
1413 	hash.w[2] ^= rol32(hash.w[2], 16);
1414 
1415 	memcpy(out, &hash, EXTRACT_SIZE);
1416 	memzero_explicit(&hash, sizeof(hash));
1417 }
1418 
1419 static ssize_t _extract_entropy(struct entropy_store *r, void *buf,
1420 				size_t nbytes, int fips)
1421 {
1422 	ssize_t ret = 0, i;
1423 	__u8 tmp[EXTRACT_SIZE];
1424 	unsigned long flags;
1425 
1426 	while (nbytes) {
1427 		extract_buf(r, tmp);
1428 
1429 		if (fips) {
1430 			spin_lock_irqsave(&r->lock, flags);
1431 			if (!memcmp(tmp, r->last_data, EXTRACT_SIZE))
1432 				panic("Hardware RNG duplicated output!\n");
1433 			memcpy(r->last_data, tmp, EXTRACT_SIZE);
1434 			spin_unlock_irqrestore(&r->lock, flags);
1435 		}
1436 		i = min_t(int, nbytes, EXTRACT_SIZE);
1437 		memcpy(buf, tmp, i);
1438 		nbytes -= i;
1439 		buf += i;
1440 		ret += i;
1441 	}
1442 
1443 	/* Wipe data just returned from memory */
1444 	memzero_explicit(tmp, sizeof(tmp));
1445 
1446 	return ret;
1447 }
1448 
1449 /*
1450  * This function extracts randomness from the "entropy pool", and
1451  * returns it in a buffer.
1452  *
1453  * The min parameter specifies the minimum amount we can pull before
1454  * failing to avoid races that defeat catastrophic reseeding while the
1455  * reserved parameter indicates how much entropy we must leave in the
1456  * pool after each pull to avoid starving other readers.
1457  */
1458 static ssize_t extract_entropy(struct entropy_store *r, void *buf,
1459 				 size_t nbytes, int min, int reserved)
1460 {
1461 	__u8 tmp[EXTRACT_SIZE];
1462 	unsigned long flags;
1463 
1464 	/* if last_data isn't primed, we need EXTRACT_SIZE extra bytes */
1465 	if (fips_enabled) {
1466 		spin_lock_irqsave(&r->lock, flags);
1467 		if (!r->last_data_init) {
1468 			r->last_data_init = 1;
1469 			spin_unlock_irqrestore(&r->lock, flags);
1470 			trace_extract_entropy(r->name, EXTRACT_SIZE,
1471 					      ENTROPY_BITS(r), _RET_IP_);
1472 			extract_buf(r, tmp);
1473 			spin_lock_irqsave(&r->lock, flags);
1474 			memcpy(r->last_data, tmp, EXTRACT_SIZE);
1475 		}
1476 		spin_unlock_irqrestore(&r->lock, flags);
1477 	}
1478 
1479 	trace_extract_entropy(r->name, nbytes, ENTROPY_BITS(r), _RET_IP_);
1480 	nbytes = account(r, nbytes, min, reserved);
1481 
1482 	return _extract_entropy(r, buf, nbytes, fips_enabled);
1483 }
1484 
1485 #define warn_unseeded_randomness(previous) \
1486 	_warn_unseeded_randomness(__func__, (void *) _RET_IP_, (previous))
1487 
1488 static void _warn_unseeded_randomness(const char *func_name, void *caller,
1489 				      void **previous)
1490 {
1491 #ifdef CONFIG_WARN_ALL_UNSEEDED_RANDOM
1492 	const bool print_once = false;
1493 #else
1494 	static bool print_once __read_mostly;
1495 #endif
1496 
1497 	if (print_once ||
1498 	    crng_ready() ||
1499 	    (previous && (caller == READ_ONCE(*previous))))
1500 		return;
1501 	WRITE_ONCE(*previous, caller);
1502 #ifndef CONFIG_WARN_ALL_UNSEEDED_RANDOM
1503 	print_once = true;
1504 #endif
1505 	if (__ratelimit(&unseeded_warning))
1506 		printk_deferred(KERN_NOTICE "random: %s called from %pS "
1507 				"with crng_init=%d\n", func_name, caller,
1508 				crng_init);
1509 }
1510 
1511 /*
1512  * This function is the exported kernel interface.  It returns some
1513  * number of good random numbers, suitable for key generation, seeding
1514  * TCP sequence numbers, etc.  It does not rely on the hardware random
1515  * number generator.  For random bytes direct from the hardware RNG
1516  * (when available), use get_random_bytes_arch(). In order to ensure
1517  * that the randomness provided by this function is okay, the function
1518  * wait_for_random_bytes() should be called and return 0 at least once
1519  * at any point prior.
1520  */
1521 static void _get_random_bytes(void *buf, int nbytes)
1522 {
1523 	__u8 tmp[CHACHA_BLOCK_SIZE] __aligned(4);
1524 
1525 	trace_get_random_bytes(nbytes, _RET_IP_);
1526 
1527 	while (nbytes >= CHACHA_BLOCK_SIZE) {
1528 		extract_crng(buf);
1529 		buf += CHACHA_BLOCK_SIZE;
1530 		nbytes -= CHACHA_BLOCK_SIZE;
1531 	}
1532 
1533 	if (nbytes > 0) {
1534 		extract_crng(tmp);
1535 		memcpy(buf, tmp, nbytes);
1536 		crng_backtrack_protect(tmp, nbytes);
1537 	} else
1538 		crng_backtrack_protect(tmp, CHACHA_BLOCK_SIZE);
1539 	memzero_explicit(tmp, sizeof(tmp));
1540 }
1541 
1542 void get_random_bytes(void *buf, int nbytes)
1543 {
1544 	static void *previous;
1545 
1546 	warn_unseeded_randomness(&previous);
1547 	_get_random_bytes(buf, nbytes);
1548 }
1549 EXPORT_SYMBOL(get_random_bytes);
1550 
1551 
1552 /*
1553  * Each time the timer fires, we expect that we got an unpredictable
1554  * jump in the cycle counter. Even if the timer is running on another
1555  * CPU, the timer activity will be touching the stack of the CPU that is
1556  * generating entropy..
1557  *
1558  * Note that we don't re-arm the timer in the timer itself - we are
1559  * happy to be scheduled away, since that just makes the load more
1560  * complex, but we do not want the timer to keep ticking unless the
1561  * entropy loop is running.
1562  *
1563  * So the re-arming always happens in the entropy loop itself.
1564  */
1565 static void entropy_timer(struct timer_list *t)
1566 {
1567 	credit_entropy_bits(&input_pool, 1);
1568 }
1569 
1570 /*
1571  * If we have an actual cycle counter, see if we can
1572  * generate enough entropy with timing noise
1573  */
1574 static void try_to_generate_entropy(void)
1575 {
1576 	struct {
1577 		unsigned long now;
1578 		struct timer_list timer;
1579 	} stack;
1580 
1581 	stack.now = random_get_entropy();
1582 
1583 	/* Slow counter - or none. Don't even bother */
1584 	if (stack.now == random_get_entropy())
1585 		return;
1586 
1587 	timer_setup_on_stack(&stack.timer, entropy_timer, 0);
1588 	while (!crng_ready()) {
1589 		if (!timer_pending(&stack.timer))
1590 			mod_timer(&stack.timer, jiffies+1);
1591 		mix_pool_bytes(&input_pool, &stack.now, sizeof(stack.now));
1592 		schedule();
1593 		stack.now = random_get_entropy();
1594 	}
1595 
1596 	del_timer_sync(&stack.timer);
1597 	destroy_timer_on_stack(&stack.timer);
1598 	mix_pool_bytes(&input_pool, &stack.now, sizeof(stack.now));
1599 }
1600 
1601 /*
1602  * Wait for the urandom pool to be seeded and thus guaranteed to supply
1603  * cryptographically secure random numbers. This applies to: the /dev/urandom
1604  * device, the get_random_bytes function, and the get_random_{u32,u64,int,long}
1605  * family of functions. Using any of these functions without first calling
1606  * this function forfeits the guarantee of security.
1607  *
1608  * Returns: 0 if the urandom pool has been seeded.
1609  *          -ERESTARTSYS if the function was interrupted by a signal.
1610  */
1611 int wait_for_random_bytes(void)
1612 {
1613 	if (likely(crng_ready()))
1614 		return 0;
1615 
1616 	do {
1617 		int ret;
1618 		ret = wait_event_interruptible_timeout(crng_init_wait, crng_ready(), HZ);
1619 		if (ret)
1620 			return ret > 0 ? 0 : ret;
1621 
1622 		try_to_generate_entropy();
1623 	} while (!crng_ready());
1624 
1625 	return 0;
1626 }
1627 EXPORT_SYMBOL(wait_for_random_bytes);
1628 
1629 /*
1630  * Returns whether or not the urandom pool has been seeded and thus guaranteed
1631  * to supply cryptographically secure random numbers. This applies to: the
1632  * /dev/urandom device, the get_random_bytes function, and the get_random_{u32,
1633  * ,u64,int,long} family of functions.
1634  *
1635  * Returns: true if the urandom pool has been seeded.
1636  *          false if the urandom pool has not been seeded.
1637  */
1638 bool rng_is_initialized(void)
1639 {
1640 	return crng_ready();
1641 }
1642 EXPORT_SYMBOL(rng_is_initialized);
1643 
1644 /*
1645  * Add a callback function that will be invoked when the nonblocking
1646  * pool is initialised.
1647  *
1648  * returns: 0 if callback is successfully added
1649  *	    -EALREADY if pool is already initialised (callback not called)
1650  *	    -ENOENT if module for callback is not alive
1651  */
1652 int add_random_ready_callback(struct random_ready_callback *rdy)
1653 {
1654 	struct module *owner;
1655 	unsigned long flags;
1656 	int err = -EALREADY;
1657 
1658 	if (crng_ready())
1659 		return err;
1660 
1661 	owner = rdy->owner;
1662 	if (!try_module_get(owner))
1663 		return -ENOENT;
1664 
1665 	spin_lock_irqsave(&random_ready_list_lock, flags);
1666 	if (crng_ready())
1667 		goto out;
1668 
1669 	owner = NULL;
1670 
1671 	list_add(&rdy->list, &random_ready_list);
1672 	err = 0;
1673 
1674 out:
1675 	spin_unlock_irqrestore(&random_ready_list_lock, flags);
1676 
1677 	module_put(owner);
1678 
1679 	return err;
1680 }
1681 EXPORT_SYMBOL(add_random_ready_callback);
1682 
1683 /*
1684  * Delete a previously registered readiness callback function.
1685  */
1686 void del_random_ready_callback(struct random_ready_callback *rdy)
1687 {
1688 	unsigned long flags;
1689 	struct module *owner = NULL;
1690 
1691 	spin_lock_irqsave(&random_ready_list_lock, flags);
1692 	if (!list_empty(&rdy->list)) {
1693 		list_del_init(&rdy->list);
1694 		owner = rdy->owner;
1695 	}
1696 	spin_unlock_irqrestore(&random_ready_list_lock, flags);
1697 
1698 	module_put(owner);
1699 }
1700 EXPORT_SYMBOL(del_random_ready_callback);
1701 
1702 /*
1703  * This function will use the architecture-specific hardware random
1704  * number generator if it is available.  The arch-specific hw RNG will
1705  * almost certainly be faster than what we can do in software, but it
1706  * is impossible to verify that it is implemented securely (as
1707  * opposed, to, say, the AES encryption of a sequence number using a
1708  * key known by the NSA).  So it's useful if we need the speed, but
1709  * only if we're willing to trust the hardware manufacturer not to
1710  * have put in a back door.
1711  *
1712  * Return number of bytes filled in.
1713  */
1714 int __must_check get_random_bytes_arch(void *buf, int nbytes)
1715 {
1716 	int left = nbytes;
1717 	char *p = buf;
1718 
1719 	trace_get_random_bytes_arch(left, _RET_IP_);
1720 	while (left) {
1721 		unsigned long v;
1722 		int chunk = min_t(int, left, sizeof(unsigned long));
1723 
1724 		if (!arch_get_random_long(&v))
1725 			break;
1726 
1727 		memcpy(p, &v, chunk);
1728 		p += chunk;
1729 		left -= chunk;
1730 	}
1731 
1732 	return nbytes - left;
1733 }
1734 EXPORT_SYMBOL(get_random_bytes_arch);
1735 
1736 /*
1737  * init_std_data - initialize pool with system data
1738  *
1739  * @r: pool to initialize
1740  *
1741  * This function clears the pool's entropy count and mixes some system
1742  * data into the pool to prepare it for use. The pool is not cleared
1743  * as that can only decrease the entropy in the pool.
1744  */
1745 static void __init init_std_data(struct entropy_store *r)
1746 {
1747 	int i;
1748 	ktime_t now = ktime_get_real();
1749 	unsigned long rv;
1750 
1751 	mix_pool_bytes(r, &now, sizeof(now));
1752 	for (i = r->poolinfo->poolbytes; i > 0; i -= sizeof(rv)) {
1753 		if (!arch_get_random_seed_long(&rv) &&
1754 		    !arch_get_random_long(&rv))
1755 			rv = random_get_entropy();
1756 		mix_pool_bytes(r, &rv, sizeof(rv));
1757 	}
1758 	mix_pool_bytes(r, utsname(), sizeof(*(utsname())));
1759 }
1760 
1761 /*
1762  * Note that setup_arch() may call add_device_randomness()
1763  * long before we get here. This allows seeding of the pools
1764  * with some platform dependent data very early in the boot
1765  * process. But it limits our options here. We must use
1766  * statically allocated structures that already have all
1767  * initializations complete at compile time. We should also
1768  * take care not to overwrite the precious per platform data
1769  * we were given.
1770  */
1771 int __init rand_initialize(void)
1772 {
1773 	init_std_data(&input_pool);
1774 	crng_initialize(&primary_crng);
1775 	crng_global_init_time = jiffies;
1776 	if (ratelimit_disable) {
1777 		urandom_warning.interval = 0;
1778 		unseeded_warning.interval = 0;
1779 	}
1780 	return 0;
1781 }
1782 
1783 #ifdef CONFIG_BLOCK
1784 void rand_initialize_disk(struct gendisk *disk)
1785 {
1786 	struct timer_rand_state *state;
1787 
1788 	/*
1789 	 * If kzalloc returns null, we just won't use that entropy
1790 	 * source.
1791 	 */
1792 	state = kzalloc(sizeof(struct timer_rand_state), GFP_KERNEL);
1793 	if (state) {
1794 		state->last_time = INITIAL_JIFFIES;
1795 		disk->random = state;
1796 	}
1797 }
1798 #endif
1799 
1800 static ssize_t
1801 urandom_read_nowarn(struct file *file, char __user *buf, size_t nbytes,
1802 		    loff_t *ppos)
1803 {
1804 	int ret;
1805 
1806 	nbytes = min_t(size_t, nbytes, INT_MAX >> (ENTROPY_SHIFT + 3));
1807 	ret = extract_crng_user(buf, nbytes);
1808 	trace_urandom_read(8 * nbytes, 0, ENTROPY_BITS(&input_pool));
1809 	return ret;
1810 }
1811 
1812 static ssize_t
1813 urandom_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos)
1814 {
1815 	unsigned long flags;
1816 	static int maxwarn = 10;
1817 
1818 	if (!crng_ready() && maxwarn > 0) {
1819 		maxwarn--;
1820 		if (__ratelimit(&urandom_warning))
1821 			pr_notice("%s: uninitialized urandom read (%zd bytes read)\n",
1822 				  current->comm, nbytes);
1823 		spin_lock_irqsave(&primary_crng.lock, flags);
1824 		crng_init_cnt = 0;
1825 		spin_unlock_irqrestore(&primary_crng.lock, flags);
1826 	}
1827 
1828 	return urandom_read_nowarn(file, buf, nbytes, ppos);
1829 }
1830 
1831 static ssize_t
1832 random_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos)
1833 {
1834 	int ret;
1835 
1836 	ret = wait_for_random_bytes();
1837 	if (ret != 0)
1838 		return ret;
1839 	return urandom_read_nowarn(file, buf, nbytes, ppos);
1840 }
1841 
1842 static __poll_t
1843 random_poll(struct file *file, poll_table * wait)
1844 {
1845 	__poll_t mask;
1846 
1847 	poll_wait(file, &crng_init_wait, wait);
1848 	poll_wait(file, &random_write_wait, wait);
1849 	mask = 0;
1850 	if (crng_ready())
1851 		mask |= EPOLLIN | EPOLLRDNORM;
1852 	if (ENTROPY_BITS(&input_pool) < random_write_wakeup_bits)
1853 		mask |= EPOLLOUT | EPOLLWRNORM;
1854 	return mask;
1855 }
1856 
1857 static int
1858 write_pool(struct entropy_store *r, const char __user *buffer, size_t count)
1859 {
1860 	size_t bytes;
1861 	__u32 t, buf[16];
1862 	const char __user *p = buffer;
1863 
1864 	while (count > 0) {
1865 		int b, i = 0;
1866 
1867 		bytes = min(count, sizeof(buf));
1868 		if (copy_from_user(&buf, p, bytes))
1869 			return -EFAULT;
1870 
1871 		for (b = bytes ; b > 0 ; b -= sizeof(__u32), i++) {
1872 			if (!arch_get_random_int(&t))
1873 				break;
1874 			buf[i] ^= t;
1875 		}
1876 
1877 		count -= bytes;
1878 		p += bytes;
1879 
1880 		mix_pool_bytes(r, buf, bytes);
1881 		cond_resched();
1882 	}
1883 
1884 	return 0;
1885 }
1886 
1887 static ssize_t random_write(struct file *file, const char __user *buffer,
1888 			    size_t count, loff_t *ppos)
1889 {
1890 	size_t ret;
1891 
1892 	ret = write_pool(&input_pool, buffer, count);
1893 	if (ret)
1894 		return ret;
1895 
1896 	return (ssize_t)count;
1897 }
1898 
1899 static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
1900 {
1901 	int size, ent_count;
1902 	int __user *p = (int __user *)arg;
1903 	int retval;
1904 
1905 	switch (cmd) {
1906 	case RNDGETENTCNT:
1907 		/* inherently racy, no point locking */
1908 		ent_count = ENTROPY_BITS(&input_pool);
1909 		if (put_user(ent_count, p))
1910 			return -EFAULT;
1911 		return 0;
1912 	case RNDADDTOENTCNT:
1913 		if (!capable(CAP_SYS_ADMIN))
1914 			return -EPERM;
1915 		if (get_user(ent_count, p))
1916 			return -EFAULT;
1917 		return credit_entropy_bits_safe(&input_pool, ent_count);
1918 	case RNDADDENTROPY:
1919 		if (!capable(CAP_SYS_ADMIN))
1920 			return -EPERM;
1921 		if (get_user(ent_count, p++))
1922 			return -EFAULT;
1923 		if (ent_count < 0)
1924 			return -EINVAL;
1925 		if (get_user(size, p++))
1926 			return -EFAULT;
1927 		retval = write_pool(&input_pool, (const char __user *)p,
1928 				    size);
1929 		if (retval < 0)
1930 			return retval;
1931 		return credit_entropy_bits_safe(&input_pool, ent_count);
1932 	case RNDZAPENTCNT:
1933 	case RNDCLEARPOOL:
1934 		/*
1935 		 * Clear the entropy pool counters. We no longer clear
1936 		 * the entropy pool, as that's silly.
1937 		 */
1938 		if (!capable(CAP_SYS_ADMIN))
1939 			return -EPERM;
1940 		input_pool.entropy_count = 0;
1941 		return 0;
1942 	case RNDRESEEDCRNG:
1943 		if (!capable(CAP_SYS_ADMIN))
1944 			return -EPERM;
1945 		if (crng_init < 2)
1946 			return -ENODATA;
1947 		crng_reseed(&primary_crng, NULL);
1948 		crng_global_init_time = jiffies - 1;
1949 		return 0;
1950 	default:
1951 		return -EINVAL;
1952 	}
1953 }
1954 
1955 static int random_fasync(int fd, struct file *filp, int on)
1956 {
1957 	return fasync_helper(fd, filp, on, &fasync);
1958 }
1959 
1960 const struct file_operations random_fops = {
1961 	.read  = random_read,
1962 	.write = random_write,
1963 	.poll  = random_poll,
1964 	.unlocked_ioctl = random_ioctl,
1965 	.compat_ioctl = compat_ptr_ioctl,
1966 	.fasync = random_fasync,
1967 	.llseek = noop_llseek,
1968 };
1969 
1970 const struct file_operations urandom_fops = {
1971 	.read  = urandom_read,
1972 	.write = random_write,
1973 	.unlocked_ioctl = random_ioctl,
1974 	.compat_ioctl = compat_ptr_ioctl,
1975 	.fasync = random_fasync,
1976 	.llseek = noop_llseek,
1977 };
1978 
1979 SYSCALL_DEFINE3(getrandom, char __user *, buf, size_t, count,
1980 		unsigned int, flags)
1981 {
1982 	int ret;
1983 
1984 	if (flags & ~(GRND_NONBLOCK|GRND_RANDOM|GRND_INSECURE))
1985 		return -EINVAL;
1986 
1987 	/*
1988 	 * Requesting insecure and blocking randomness at the same time makes
1989 	 * no sense.
1990 	 */
1991 	if ((flags & (GRND_INSECURE|GRND_RANDOM)) == (GRND_INSECURE|GRND_RANDOM))
1992 		return -EINVAL;
1993 
1994 	if (count > INT_MAX)
1995 		count = INT_MAX;
1996 
1997 	if (!(flags & GRND_INSECURE) && !crng_ready()) {
1998 		if (flags & GRND_NONBLOCK)
1999 			return -EAGAIN;
2000 		ret = wait_for_random_bytes();
2001 		if (unlikely(ret))
2002 			return ret;
2003 	}
2004 	return urandom_read_nowarn(NULL, buf, count, NULL);
2005 }
2006 
2007 /********************************************************************
2008  *
2009  * Sysctl interface
2010  *
2011  ********************************************************************/
2012 
2013 #ifdef CONFIG_SYSCTL
2014 
2015 #include <linux/sysctl.h>
2016 
2017 static int min_write_thresh;
2018 static int max_write_thresh = INPUT_POOL_WORDS * 32;
2019 static int random_min_urandom_seed = 60;
2020 static char sysctl_bootid[16];
2021 
2022 /*
2023  * This function is used to return both the bootid UUID, and random
2024  * UUID.  The difference is in whether table->data is NULL; if it is,
2025  * then a new UUID is generated and returned to the user.
2026  *
2027  * If the user accesses this via the proc interface, the UUID will be
2028  * returned as an ASCII string in the standard UUID format; if via the
2029  * sysctl system call, as 16 bytes of binary data.
2030  */
2031 static int proc_do_uuid(struct ctl_table *table, int write,
2032 			void __user *buffer, size_t *lenp, loff_t *ppos)
2033 {
2034 	struct ctl_table fake_table;
2035 	unsigned char buf[64], tmp_uuid[16], *uuid;
2036 
2037 	uuid = table->data;
2038 	if (!uuid) {
2039 		uuid = tmp_uuid;
2040 		generate_random_uuid(uuid);
2041 	} else {
2042 		static DEFINE_SPINLOCK(bootid_spinlock);
2043 
2044 		spin_lock(&bootid_spinlock);
2045 		if (!uuid[8])
2046 			generate_random_uuid(uuid);
2047 		spin_unlock(&bootid_spinlock);
2048 	}
2049 
2050 	sprintf(buf, "%pU", uuid);
2051 
2052 	fake_table.data = buf;
2053 	fake_table.maxlen = sizeof(buf);
2054 
2055 	return proc_dostring(&fake_table, write, buffer, lenp, ppos);
2056 }
2057 
2058 /*
2059  * Return entropy available scaled to integral bits
2060  */
2061 static int proc_do_entropy(struct ctl_table *table, int write,
2062 			   void __user *buffer, size_t *lenp, loff_t *ppos)
2063 {
2064 	struct ctl_table fake_table;
2065 	int entropy_count;
2066 
2067 	entropy_count = *(int *)table->data >> ENTROPY_SHIFT;
2068 
2069 	fake_table.data = &entropy_count;
2070 	fake_table.maxlen = sizeof(entropy_count);
2071 
2072 	return proc_dointvec(&fake_table, write, buffer, lenp, ppos);
2073 }
2074 
2075 static int sysctl_poolsize = INPUT_POOL_WORDS * 32;
2076 extern struct ctl_table random_table[];
2077 struct ctl_table random_table[] = {
2078 	{
2079 		.procname	= "poolsize",
2080 		.data		= &sysctl_poolsize,
2081 		.maxlen		= sizeof(int),
2082 		.mode		= 0444,
2083 		.proc_handler	= proc_dointvec,
2084 	},
2085 	{
2086 		.procname	= "entropy_avail",
2087 		.maxlen		= sizeof(int),
2088 		.mode		= 0444,
2089 		.proc_handler	= proc_do_entropy,
2090 		.data		= &input_pool.entropy_count,
2091 	},
2092 	{
2093 		.procname	= "write_wakeup_threshold",
2094 		.data		= &random_write_wakeup_bits,
2095 		.maxlen		= sizeof(int),
2096 		.mode		= 0644,
2097 		.proc_handler	= proc_dointvec_minmax,
2098 		.extra1		= &min_write_thresh,
2099 		.extra2		= &max_write_thresh,
2100 	},
2101 	{
2102 		.procname	= "urandom_min_reseed_secs",
2103 		.data		= &random_min_urandom_seed,
2104 		.maxlen		= sizeof(int),
2105 		.mode		= 0644,
2106 		.proc_handler	= proc_dointvec,
2107 	},
2108 	{
2109 		.procname	= "boot_id",
2110 		.data		= &sysctl_bootid,
2111 		.maxlen		= 16,
2112 		.mode		= 0444,
2113 		.proc_handler	= proc_do_uuid,
2114 	},
2115 	{
2116 		.procname	= "uuid",
2117 		.maxlen		= 16,
2118 		.mode		= 0444,
2119 		.proc_handler	= proc_do_uuid,
2120 	},
2121 #ifdef ADD_INTERRUPT_BENCH
2122 	{
2123 		.procname	= "add_interrupt_avg_cycles",
2124 		.data		= &avg_cycles,
2125 		.maxlen		= sizeof(avg_cycles),
2126 		.mode		= 0444,
2127 		.proc_handler	= proc_doulongvec_minmax,
2128 	},
2129 	{
2130 		.procname	= "add_interrupt_avg_deviation",
2131 		.data		= &avg_deviation,
2132 		.maxlen		= sizeof(avg_deviation),
2133 		.mode		= 0444,
2134 		.proc_handler	= proc_doulongvec_minmax,
2135 	},
2136 #endif
2137 	{ }
2138 };
2139 #endif 	/* CONFIG_SYSCTL */
2140 
2141 struct batched_entropy {
2142 	union {
2143 		u64 entropy_u64[CHACHA_BLOCK_SIZE / sizeof(u64)];
2144 		u32 entropy_u32[CHACHA_BLOCK_SIZE / sizeof(u32)];
2145 	};
2146 	unsigned int position;
2147 	spinlock_t batch_lock;
2148 };
2149 
2150 /*
2151  * Get a random word for internal kernel use only. The quality of the random
2152  * number is either as good as RDRAND or as good as /dev/urandom, with the
2153  * goal of being quite fast and not depleting entropy. In order to ensure
2154  * that the randomness provided by this function is okay, the function
2155  * wait_for_random_bytes() should be called and return 0 at least once
2156  * at any point prior.
2157  */
2158 static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = {
2159 	.batch_lock	= __SPIN_LOCK_UNLOCKED(batched_entropy_u64.lock),
2160 };
2161 
2162 u64 get_random_u64(void)
2163 {
2164 	u64 ret;
2165 	unsigned long flags;
2166 	struct batched_entropy *batch;
2167 	static void *previous;
2168 
2169 #if BITS_PER_LONG == 64
2170 	if (arch_get_random_long((unsigned long *)&ret))
2171 		return ret;
2172 #else
2173 	if (arch_get_random_long((unsigned long *)&ret) &&
2174 	    arch_get_random_long((unsigned long *)&ret + 1))
2175 	    return ret;
2176 #endif
2177 
2178 	warn_unseeded_randomness(&previous);
2179 
2180 	batch = raw_cpu_ptr(&batched_entropy_u64);
2181 	spin_lock_irqsave(&batch->batch_lock, flags);
2182 	if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) {
2183 		extract_crng((u8 *)batch->entropy_u64);
2184 		batch->position = 0;
2185 	}
2186 	ret = batch->entropy_u64[batch->position++];
2187 	spin_unlock_irqrestore(&batch->batch_lock, flags);
2188 	return ret;
2189 }
2190 EXPORT_SYMBOL(get_random_u64);
2191 
2192 static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32) = {
2193 	.batch_lock	= __SPIN_LOCK_UNLOCKED(batched_entropy_u32.lock),
2194 };
2195 u32 get_random_u32(void)
2196 {
2197 	u32 ret;
2198 	unsigned long flags;
2199 	struct batched_entropy *batch;
2200 	static void *previous;
2201 
2202 	if (arch_get_random_int(&ret))
2203 		return ret;
2204 
2205 	warn_unseeded_randomness(&previous);
2206 
2207 	batch = raw_cpu_ptr(&batched_entropy_u32);
2208 	spin_lock_irqsave(&batch->batch_lock, flags);
2209 	if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) {
2210 		extract_crng((u8 *)batch->entropy_u32);
2211 		batch->position = 0;
2212 	}
2213 	ret = batch->entropy_u32[batch->position++];
2214 	spin_unlock_irqrestore(&batch->batch_lock, flags);
2215 	return ret;
2216 }
2217 EXPORT_SYMBOL(get_random_u32);
2218 
2219 /* It's important to invalidate all potential batched entropy that might
2220  * be stored before the crng is initialized, which we can do lazily by
2221  * simply resetting the counter to zero so that it's re-extracted on the
2222  * next usage. */
2223 static void invalidate_batched_entropy(void)
2224 {
2225 	int cpu;
2226 	unsigned long flags;
2227 
2228 	for_each_possible_cpu (cpu) {
2229 		struct batched_entropy *batched_entropy;
2230 
2231 		batched_entropy = per_cpu_ptr(&batched_entropy_u32, cpu);
2232 		spin_lock_irqsave(&batched_entropy->batch_lock, flags);
2233 		batched_entropy->position = 0;
2234 		spin_unlock(&batched_entropy->batch_lock);
2235 
2236 		batched_entropy = per_cpu_ptr(&batched_entropy_u64, cpu);
2237 		spin_lock(&batched_entropy->batch_lock);
2238 		batched_entropy->position = 0;
2239 		spin_unlock_irqrestore(&batched_entropy->batch_lock, flags);
2240 	}
2241 }
2242 
2243 /**
2244  * randomize_page - Generate a random, page aligned address
2245  * @start:	The smallest acceptable address the caller will take.
2246  * @range:	The size of the area, starting at @start, within which the
2247  *		random address must fall.
2248  *
2249  * If @start + @range would overflow, @range is capped.
2250  *
2251  * NOTE: Historical use of randomize_range, which this replaces, presumed that
2252  * @start was already page aligned.  We now align it regardless.
2253  *
2254  * Return: A page aligned address within [start, start + range).  On error,
2255  * @start is returned.
2256  */
2257 unsigned long
2258 randomize_page(unsigned long start, unsigned long range)
2259 {
2260 	if (!PAGE_ALIGNED(start)) {
2261 		range -= PAGE_ALIGN(start) - start;
2262 		start = PAGE_ALIGN(start);
2263 	}
2264 
2265 	if (start > ULONG_MAX - range)
2266 		range = ULONG_MAX - start;
2267 
2268 	range >>= PAGE_SHIFT;
2269 
2270 	if (range == 0)
2271 		return start;
2272 
2273 	return start + (get_random_long() % range << PAGE_SHIFT);
2274 }
2275 
2276 /* Interface for in-kernel drivers of true hardware RNGs.
2277  * Those devices may produce endless random bits and will be throttled
2278  * when our pool is full.
2279  */
2280 void add_hwgenerator_randomness(const char *buffer, size_t count,
2281 				size_t entropy)
2282 {
2283 	struct entropy_store *poolp = &input_pool;
2284 
2285 	if (unlikely(crng_init == 0)) {
2286 		crng_fast_load(buffer, count);
2287 		return;
2288 	}
2289 
2290 	/* Suspend writing if we're above the trickle threshold.
2291 	 * We'll be woken up again once below random_write_wakeup_thresh,
2292 	 * or when the calling thread is about to terminate.
2293 	 */
2294 	wait_event_interruptible(random_write_wait, kthread_should_stop() ||
2295 			ENTROPY_BITS(&input_pool) <= random_write_wakeup_bits);
2296 	mix_pool_bytes(poolp, buffer, count);
2297 	credit_entropy_bits(poolp, entropy);
2298 }
2299 EXPORT_SYMBOL_GPL(add_hwgenerator_randomness);
2300 
2301 /* Handle random seed passed by bootloader.
2302  * If the seed is trustworthy, it would be regarded as hardware RNGs. Otherwise
2303  * it would be regarded as device data.
2304  * The decision is controlled by CONFIG_RANDOM_TRUST_BOOTLOADER.
2305  */
2306 void add_bootloader_randomness(const void *buf, unsigned int size)
2307 {
2308 	if (IS_ENABLED(CONFIG_RANDOM_TRUST_BOOTLOADER))
2309 		add_hwgenerator_randomness(buf, size, size * 8);
2310 	else
2311 		add_device_randomness(buf, size);
2312 }
2313 EXPORT_SYMBOL_GPL(add_bootloader_randomness);
2314