1/*
   2 * random.c -- A strong random number generator
   3 *
   4 * Copyright (C) 2017 Jason A. Donenfeld <Jason@zx2c4.com>. All
   5 * Rights Reserved.
   6 *
   7 * Copyright Matt Mackall <mpm@selenic.com>, 2003, 2004, 2005
   8 *
   9 * Copyright Theodore Ts'o, 1994, 1995, 1996, 1997, 1998, 1999.  All
  10 * rights reserved.
  11 *
  12 * Redistribution and use in source and binary forms, with or without
  13 * modification, are permitted provided that the following conditions
  14 * are met:
  15 * 1. Redistributions of source code must retain the above copyright
  16 *    notice, and the entire permission notice in its entirety,
  17 *    including the disclaimer of warranties.
  18 * 2. Redistributions in binary form must reproduce the above copyright
  19 *    notice, this list of conditions and the following disclaimer in the
  20 *    documentation and/or other materials provided with the distribution.
  21 * 3. The name of the author may not be used to endorse or promote
  22 *    products derived from this software without specific prior
  23 *    written permission.
  24 *
  25 * ALTERNATIVELY, this product may be distributed under the terms of
  26 * the GNU General Public License, in which case the provisions of the GPL are
  27 * required INSTEAD OF the above restrictions.  (This clause is
  28 * necessary due to a potential bad interaction between the GPL and
  29 * the restrictions contained in a BSD-style copyright.)
  30 *
  31 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
  32 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  33 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
  34 * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
  35 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  36 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
  37 * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  38 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  39 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
  41 * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
  42 * DAMAGE.
  43 */
  44
  45/*
  46 * (now, with legal B.S. out of the way.....)
  47 *
  48 * This routine gathers environmental noise from device drivers, etc.,
  49 * and returns good random numbers, suitable for cryptographic use.
  50 * Besides the obvious cryptographic uses, these numbers are also good
  51 * for seeding TCP sequence numbers, and other places where it is
  52 * desirable to have numbers which are not only random, but hard to
  53 * predict by an attacker.
  54 *
  55 * Theory of operation
  56 * ===================
  57 *
  58 * Computers are very predictable devices.  Hence it is extremely hard
  59 * to produce truly random numbers on a computer --- as opposed to
  60 * pseudo-random numbers, which can easily generated by using a
  61 * algorithm.  Unfortunately, it is very easy for attackers to guess
  62 * the sequence of pseudo-random number generators, and for some
  63 * applications this is not acceptable.  So instead, we must try to
  64 * gather "environmental noise" from the computer's environment, which
  65 * must be hard for outside attackers to observe, and use that to
  66 * generate random numbers.  In a Unix environment, this is best done
  67 * from inside the kernel.
  68 *
  69 * Sources of randomness from the environment include inter-keyboard
  70 * timings, inter-interrupt timings from some interrupts, and other
  71 * events which are both (a) non-deterministic and (b) hard for an
  72 * outside observer to measure.  Randomness from these sources are
  73 * added to an "entropy pool", which is mixed using a CRC-like function.
  74 * This is not cryptographically strong, but it is adequate assuming
  75 * the randomness is not chosen maliciously, and it is fast enough that
  76 * the overhead of doing it on every interrupt is very reasonable.
  77 * As random bytes are mixed into the entropy pool, the routines keep
  78 * an *estimate* of how many bits of randomness have been stored into
  79 * the random number generator's internal state.
  80 *
  81 * When random bytes are desired, they are obtained by taking the SHA
  82 * hash of the contents of the "entropy pool".  The SHA hash avoids
  83 * exposing the internal state of the entropy pool.  It is believed to
  84 * be computationally infeasible to derive any useful information
  85 * about the input of SHA from its output.  Even if it is possible to
  86 * analyze SHA in some clever way, as long as the amount of data
  87 * returned from the generator is less than the inherent entropy in
  88 * the pool, the output data is totally unpredictable.  For this
  89 * reason, the routine decreases its internal estimate of how many
  90 * bits of "true randomness" are contained in the entropy pool as it
  91 * outputs random numbers.
  92 *
  93 * If this estimate goes to zero, the routine can still generate
  94 * random numbers; however, an attacker may (at least in theory) be
  95 * able to infer the future output of the generator from prior
  96 * outputs.  This requires successful cryptanalysis of SHA, which is
  97 * not believed to be feasible, but there is a remote possibility.
  98 * Nonetheless, these numbers should be useful for the vast majority
  99 * of purposes.
 100 *
 101 * Exported interfaces ---- output
 102 * ===============================
 103 *
 104 * There are four exported interfaces; two for use within the kernel,
 105 * and two or use from userspace.
 106 *
 107 * Exported interfaces ---- userspace output
 108 * -----------------------------------------
 109 *
 110 * The userspace interfaces are two character devices /dev/random and
 111 * /dev/urandom.  /dev/random is suitable for use when very high
 112 * quality randomness is desired (for example, for key generation or
 113 * one-time pads), as it will only return a maximum of the number of
 114 * bits of randomness (as estimated by the random number generator)
 115 * contained in the entropy pool.
 116 *
 117 * The /dev/urandom device does not have this limit, and will return
 118 * as many bytes as are requested.  As more and more random bytes are
 119 * requested without giving time for the entropy pool to recharge,
 120 * this will result in random numbers that are merely cryptographically
 121 * strong.  For many applications, however, this is acceptable.
 122 *
 123 * Exported interfaces ---- kernel output
 124 * --------------------------------------
 125 *
 126 * The primary kernel interface is
 127 *
 128 * 	void get_random_bytes(void *buf, int nbytes);
 129 *
 130 * This interface will return the requested number of random bytes,
 131 * and place it in the requested buffer.  This is equivalent to a
 132 * read from /dev/urandom.
 133 *
 134 * For less critical applications, there are the functions:
 135 *
 136 * 	u32 get_random_u32()
 137 * 	u64 get_random_u64()
 138 * 	unsigned int get_random_int()
 139 * 	unsigned long get_random_long()
 140 *
 141 * These are produced by a cryptographic RNG seeded from get_random_bytes,
 142 * and so do not deplete the entropy pool as much.  These are recommended
 143 * for most in-kernel operations *if the result is going to be stored in
 144 * the kernel*.
 145 *
 146 * Specifically, the get_random_int() family do not attempt to do
 147 * "anti-backtracking".  If you capture the state of the kernel (e.g.
 148 * by snapshotting the VM), you can figure out previous get_random_int()
 149 * return values.  But if the value is stored in the kernel anyway,
 150 * this is not a problem.
 151 *
 152 * It *is* safe to expose get_random_int() output to attackers (e.g. as
 153 * network cookies); given outputs 1..n, it's not feasible to predict
 154 * outputs 0 or n+1.  The only concern is an attacker who breaks into
 155 * the kernel later; the get_random_int() engine is not reseeded as
 156 * often as the get_random_bytes() one.
 157 *
 158 * get_random_bytes() is needed for keys that need to stay secret after
 159 * they are erased from the kernel.  For example, any key that will
 160 * be wrapped and stored encrypted.  And session encryption keys: we'd
 161 * like to know that after the session is closed and the keys erased,
 162 * the plaintext is unrecoverable to someone who recorded the ciphertext.
 163 *
 164 * But for network ports/cookies, stack canaries, PRNG seeds, address
 165 * space layout randomization, session *authentication* keys, or other
 166 * applications where the sensitive data is stored in the kernel in
 167 * plaintext for as long as it's sensitive, the get_random_int() family
 168 * is just fine.
 169 *
 170 * Consider ASLR.  We want to keep the address space secret from an
 171 * outside attacker while the process is running, but once the address
 172 * space is torn down, it's of no use to an attacker any more.  And it's
 173 * stored in kernel data structures as long as it's alive, so worrying
 174 * about an attacker's ability to extrapolate it from the get_random_int()
 175 * CRNG is silly.
 176 *
 177 * Even some cryptographic keys are safe to generate with get_random_int().
 178 * In particular, keys for SipHash are generally fine.  Here, knowledge
 179 * of the key authorizes you to do something to a kernel object (inject
 180 * packets to a network connection, or flood a hash table), and the
 181 * key is stored with the object being protected.  Once it goes away,
 182 * we no longer care if anyone knows the key.
 183 *
 184 * prandom_u32()
 185 * -------------
 186 *
 187 * For even weaker applications, see the pseudorandom generator
 188 * prandom_u32(), prandom_max(), and prandom_bytes().  If the random
 189 * numbers aren't security-critical at all, these are *far* cheaper.
 190 * Useful for self-tests, random error simulation, randomized backoffs,
 191 * and any other application where you trust that nobody is trying to
 192 * maliciously mess with you by guessing the "random" numbers.
 193 *
 194 * Exported interfaces ---- input
 195 * ==============================
 196 *
 197 * The current exported interfaces for gathering environmental noise
 198 * from the devices are:
 199 *
 200 *	void add_device_randomness(const void *buf, unsigned int size);
 201 * 	void add_input_randomness(unsigned int type, unsigned int code,
 202 *                                unsigned int value);
 203 *	void add_interrupt_randomness(int irq, int irq_flags);
 204 * 	void add_disk_randomness(struct gendisk *disk);
 205 *
 206 * add_device_randomness() is for adding data to the random pool that
 207 * is likely to differ between two devices (or possibly even per boot).
 208 * This would be things like MAC addresses or serial numbers, or the
 209 * read-out of the RTC. This does *not* add any actual entropy to the
 210 * pool, but it initializes the pool to different values for devices
 211 * that might otherwise be identical and have very little entropy
 212 * available to them (particularly common in the embedded world).
 213 *
 214 * add_input_randomness() uses the input layer interrupt timing, as well as
 215 * the event type information from the hardware.
 216 *
 217 * add_interrupt_randomness() uses the interrupt timing as random
 218 * inputs to the entropy pool. Using the cycle counters and the irq source
 219 * as inputs, it feeds the randomness roughly once a second.
 220 *
 221 * add_disk_randomness() uses what amounts to the seek time of block
 222 * layer request events, on a per-disk_devt basis, as input to the
 223 * entropy pool. Note that high-speed solid state drives with very low
 224 * seek times do not make for good sources of entropy, as their seek
 225 * times are usually fairly consistent.
 226 *
 227 * All of these routines try to estimate how many bits of randomness a
 228 * particular randomness source.  They do this by keeping track of the
 229 * first and second order deltas of the event timings.
 230 *
 231 * Ensuring unpredictability at system startup
 232 * ============================================
 233 *
 234 * When any operating system starts up, it will go through a sequence
 235 * of actions that are fairly predictable by an adversary, especially
 236 * if the start-up does not involve interaction with a human operator.
 237 * This reduces the actual number of bits of unpredictability in the
 238 * entropy pool below the value in entropy_count.  In order to
 239 * counteract this effect, it helps to carry information in the
 240 * entropy pool across shut-downs and start-ups.  To do this, put the
 241 * following lines an appropriate script which is run during the boot
 242 * sequence:
 243 *
 244 *	echo "Initializing random number generator..."
 245 *	random_seed=/var/run/random-seed
 246 *	# Carry a random seed from start-up to start-up
 247 *	# Load and then save the whole entropy pool
 248 *	if [ -f $random_seed ]; then
 249 *		cat $random_seed >/dev/urandom
 250 *	else
 251 *		touch $random_seed
 252 *	fi
 253 *	chmod 600 $random_seed
 254 *	dd if=/dev/urandom of=$random_seed count=1 bs=512
 255 *
 256 * and the following lines in an appropriate script which is run as
 257 * the system is shutdown:
 258 *
 259 *	# Carry a random seed from shut-down to start-up
 260 *	# Save the whole entropy pool
 261 *	echo "Saving random seed..."
 262 *	random_seed=/var/run/random-seed
 263 *	touch $random_seed
 264 *	chmod 600 $random_seed
 265 *	dd if=/dev/urandom of=$random_seed count=1 bs=512
 266 *
 267 * For example, on most modern systems using the System V init
 268 * scripts, such code fragments would be found in
 269 * /etc/rc.d/init.d/random.  On older Linux systems, the correct script
 270 * location might be in /etc/rcb.d/rc.local or /etc/rc.d/rc.0.
 271 *
 272 * Effectively, these commands cause the contents of the entropy pool
 273 * to be saved at shut-down time and reloaded into the entropy pool at
 274 * start-up.  (The 'dd' in the addition to the bootup script is to
 275 * make sure that /etc/random-seed is different for every start-up,
 276 * even if the system crashes without executing rc.0.)  Even with
 277 * complete knowledge of the start-up activities, predicting the state
 278 * of the entropy pool requires knowledge of the previous history of
 279 * the system.
 280 *
 281 * Configuring the /dev/random driver under Linux
 282 * ==============================================
 283 *
 284 * The /dev/random driver under Linux uses minor numbers 8 and 9 of
 285 * the /dev/mem major number (#1).  So if your system does not have
 286 * /dev/random and /dev/urandom created already, they can be created
 287 * by using the commands:
 288 *
 289 * 	mknod /dev/random c 1 8
 290 * 	mknod /dev/urandom c 1 9
 291 *
 292 * Acknowledgements:
 293 * =================
 294 *
 295 * Ideas for constructing this random number generator were derived
 296 * from Pretty Good Privacy's random number generator, and from private
 297 * discussions with Phil Karn.  Colin Plumb provided a faster random
 298 * number generator, which speed up the mixing function of the entropy
 299 * pool, taken from PGPfone.  Dale Worley has also contributed many
 300 * useful ideas and suggestions to improve this driver.
 301 *
 302 * Any flaws in the design are solely my responsibility, and should
 303 * not be attributed to the Phil, Colin, or any of authors of PGP.
 304 *
 305 * Further background information on this topic may be obtained from
 306 * RFC 1750, "Randomness Recommendations for Security", by Donald
 307 * Eastlake, Steve Crocker, and Jeff Schiller.
 308 */
 309
 310#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 311
 312#include <linux/utsname.h>
 313#include <linux/module.h>
 314#include <linux/kernel.h>
 315#include <linux/major.h>
 316#include <linux/string.h>
 317#include <linux/fcntl.h>
 318#include <linux/slab.h>
 319#include <linux/random.h>
 320#include <linux/poll.h>
 321#include <linux/init.h>
 322#include <linux/fs.h>
 323#include <linux/genhd.h>
 324#include <linux/interrupt.h>
 325#include <linux/mm.h>
 326#include <linux/nodemask.h>
 327#include <linux/spinlock.h>
 328#include <linux/kthread.h>
 329#include <linux/percpu.h>
 330#include <linux/fips.h>
 331#include <linux/ptrace.h>
 332#include <linux/workqueue.h>
 333#include <linux/irq.h>
 334#include <linux/ratelimit.h>
 335#include <linux/syscalls.h>
 336#include <linux/completion.h>
 337#include <linux/uuid.h>
 338#include <crypto/chacha.h>
 339#include <crypto/sha.h>
 340
 341#include <asm/processor.h>
 342#include <linux/uaccess.h>
 343#include <asm/irq.h>
 344#include <asm/irq_regs.h>
 345#include <asm/io.h>
 346
 347#define CREATE_TRACE_POINTS
 348#include <trace/events/random.h>
 349
 350/* #define ADD_INTERRUPT_BENCH */
 351
 352/*
 353 * Configuration information
 354 */
 355#define INPUT_POOL_SHIFT	12
 356#define INPUT_POOL_WORDS	(1 << (INPUT_POOL_SHIFT-5))
 357#define OUTPUT_POOL_SHIFT	10
 358#define OUTPUT_POOL_WORDS	(1 << (OUTPUT_POOL_SHIFT-5))
 359#define EXTRACT_SIZE		10
 360
 361
 362#define LONGS(x) (((x) + sizeof(unsigned long) - 1)/sizeof(unsigned long))
 363
 364/*
 365 * To allow fractional bits to be tracked, the entropy_count field is
 366 * denominated in units of 1/8th bits.
 367 *
 368 * 2*(ENTROPY_SHIFT + poolbitshift) must <= 31, or the multiply in
 369 * credit_entropy_bits() needs to be 64 bits wide.
 370 */
 371#define ENTROPY_SHIFT 3
 372#define ENTROPY_BITS(r) ((r)->entropy_count >> ENTROPY_SHIFT)
 373
 374/*
 375 * If the entropy count falls under this number of bits, then we
 376 * should wake up processes which are selecting or polling on write
 377 * access to /dev/random.
 378 */
 379static int random_write_wakeup_bits = 28 * OUTPUT_POOL_WORDS;
 380
 381/*
 382 * Originally, we used a primitive polynomial of degree .poolwords
 383 * over GF(2).  The taps for various sizes are defined below.  They
 384 * were chosen to be evenly spaced except for the last tap, which is 1
 385 * to get the twisting happening as fast as possible.
 386 *
 387 * For the purposes of better mixing, we use the CRC-32 polynomial as
 388 * well to make a (modified) twisted Generalized Feedback Shift
 389 * Register.  (See M. Matsumoto & Y. Kurita, 1992.  Twisted GFSR
 390 * generators.  ACM Transactions on Modeling and Computer Simulation
 391 * 2(3):179-194.  Also see M. Matsumoto & Y. Kurita, 1994.  Twisted
 392 * GFSR generators II.  ACM Transactions on Modeling and Computer
 393 * Simulation 4:254-266)
 394 *
 395 * Thanks to Colin Plumb for suggesting this.
 396 *
 397 * The mixing operation is much less sensitive than the output hash,
 398 * where we use SHA-1.  All that we want of mixing operation is that
 399 * it be a good non-cryptographic hash; i.e. it not produce collisions
 400 * when fed "random" data of the sort we expect to see.  As long as
 401 * the pool state differs for different inputs, we have preserved the
 402 * input entropy and done a good job.  The fact that an intelligent
 403 * attacker can construct inputs that will produce controlled
 404 * alterations to the pool's state is not important because we don't
 405 * consider such inputs to contribute any randomness.  The only
 406 * property we need with respect to them is that the attacker can't
 407 * increase his/her knowledge of the pool's state.  Since all
 408 * additions are reversible (knowing the final state and the input,
 409 * you can reconstruct the initial state), if an attacker has any
 410 * uncertainty about the initial state, he/she can only shuffle that
 411 * uncertainty about, but never cause any collisions (which would
 412 * decrease the uncertainty).
 413 *
 414 * Our mixing functions were analyzed by Lacharme, Roeck, Strubel, and
 415 * Videau in their paper, "The Linux Pseudorandom Number Generator
 416 * Revisited" (see: http://eprint.iacr.org/2012/251.pdf).  In their
 417 * paper, they point out that we are not using a true Twisted GFSR,
 418 * since Matsumoto & Kurita used a trinomial feedback polynomial (that
 419 * is, with only three taps, instead of the six that we are using).
 420 * As a result, the resulting polynomial is neither primitive nor
 421 * irreducible, and hence does not have a maximal period over
 422 * GF(2**32).  They suggest a slight change to the generator
 423 * polynomial which improves the resulting TGFSR polynomial to be
 424 * irreducible, which we have made here.
 425 */
 426static const struct poolinfo {
 427	int poolbitshift, poolwords, poolbytes, poolfracbits;
 428#define S(x) ilog2(x)+5, (x), (x)*4, (x) << (ENTROPY_SHIFT+5)
 429	int tap1, tap2, tap3, tap4, tap5;
 430} poolinfo_table[] = {
 431	/* was: x^128 + x^103 + x^76 + x^51 +x^25 + x + 1 */
 432	/* x^128 + x^104 + x^76 + x^51 +x^25 + x + 1 */
 433	{ S(128),	104,	76,	51,	25,	1 },
 434};
 435
 436/*
 437 * Static global variables
 438 */
 439static DECLARE_WAIT_QUEUE_HEAD(random_write_wait);
 440static struct fasync_struct *fasync;
 441
 442static DEFINE_SPINLOCK(random_ready_list_lock);
 443static LIST_HEAD(random_ready_list);
 444
 445struct crng_state {
 446	__u32		state[16];
 447	unsigned long	init_time;
 448	spinlock_t	lock;
 449};
 450
 451static struct crng_state primary_crng = {
 452	.lock = __SPIN_LOCK_UNLOCKED(primary_crng.lock),
 453};
 454
 455/*
 456 * crng_init =  0 --> Uninitialized
 457 *		1 --> Initialized
 458 *		2 --> Initialized from input_pool
 459 *
 460 * crng_init is protected by primary_crng->lock, and only increases
 461 * its value (from 0->1->2).
 462 */
 463static int crng_init = 0;
 464#define crng_ready() (likely(crng_init > 1))
 465static int crng_init_cnt = 0;
 466static unsigned long crng_global_init_time = 0;
 467#define CRNG_INIT_CNT_THRESH (2*CHACHA_KEY_SIZE)
 468static void _extract_crng(struct crng_state *crng, __u8 out[CHACHA_BLOCK_SIZE]);
 469static void _crng_backtrack_protect(struct crng_state *crng,
 470				    __u8 tmp[CHACHA_BLOCK_SIZE], int used);
 471static void process_random_ready_list(void);
 472static void _get_random_bytes(void *buf, int nbytes);
 473
 474static struct ratelimit_state unseeded_warning =
 475	RATELIMIT_STATE_INIT("warn_unseeded_randomness", HZ, 3);
 476static struct ratelimit_state urandom_warning =
 477	RATELIMIT_STATE_INIT("warn_urandom_randomness", HZ, 3);
 478
 479static int ratelimit_disable __read_mostly;
 480
 481module_param_named(ratelimit_disable, ratelimit_disable, int, 0644);
 482MODULE_PARM_DESC(ratelimit_disable, "Disable random ratelimit suppression");
 483
 484/**********************************************************************
 485 *
 486 * OS independent entropy store.   Here are the functions which handle
 487 * storing entropy in an entropy pool.
 488 *
 489 **********************************************************************/
 490
 491struct entropy_store;
 492struct entropy_store {
 493	/* read-only data: */
 494	const struct poolinfo *poolinfo;
 495	__u32 *pool;
 496	const char *name;
 497
 498	/* read-write data: */
 499	spinlock_t lock;
 500	unsigned short add_ptr;
 501	unsigned short input_rotate;
 502	int entropy_count;
 503	unsigned int initialized:1;
 504	unsigned int last_data_init:1;
 505	__u8 last_data[EXTRACT_SIZE];
 506};
 507
 508static ssize_t extract_entropy(struct entropy_store *r, void *buf,
 509			       size_t nbytes, int min, int rsvd);
 510static ssize_t _extract_entropy(struct entropy_store *r, void *buf,
 511				size_t nbytes, int fips);
 512
 513static void crng_reseed(struct crng_state *crng, struct entropy_store *r);
 514static __u32 input_pool_data[INPUT_POOL_WORDS] __latent_entropy;
 515
 516static struct entropy_store input_pool = {
 517	.poolinfo = &poolinfo_table[0],
 518	.name = "input",
 519	.lock = __SPIN_LOCK_UNLOCKED(input_pool.lock),
 520	.pool = input_pool_data
 521};
 522
 523static __u32 const twist_table[8] = {
 524	0x00000000, 0x3b6e20c8, 0x76dc4190, 0x4db26158,
 525	0xedb88320, 0xd6d6a3e8, 0x9b64c2b0, 0xa00ae278 };
 526
 527/*
 528 * This function adds bytes into the entropy "pool".  It does not
 529 * update the entropy estimate.  The caller should call
 530 * credit_entropy_bits if this is appropriate.
 531 *
 532 * The pool is stirred with a primitive polynomial of the appropriate
 533 * degree, and then twisted.  We twist by three bits at a time because
 534 * it's cheap to do so and helps slightly in the expected case where
 535 * the entropy is concentrated in the low-order bits.
 536 */
 537static void _mix_pool_bytes(struct entropy_store *r, const void *in,
 538			    int nbytes)
 539{
 540	unsigned long i, tap1, tap2, tap3, tap4, tap5;
 541	int input_rotate;
 542	int wordmask = r->poolinfo->poolwords - 1;
 543	const char *bytes = in;
 544	__u32 w;
 545
 546	tap1 = r->poolinfo->tap1;
 547	tap2 = r->poolinfo->tap2;
 548	tap3 = r->poolinfo->tap3;
 549	tap4 = r->poolinfo->tap4;
 550	tap5 = r->poolinfo->tap5;
 551
 552	input_rotate = r->input_rotate;
 553	i = r->add_ptr;
 554
 555	/* mix one byte at a time to simplify size handling and churn faster */
 556	while (nbytes--) {
 557		w = rol32(*bytes++, input_rotate);
 558		i = (i - 1) & wordmask;
 559
 560		/* XOR in the various taps */
 561		w ^= r->pool[i];
 562		w ^= r->pool[(i + tap1) & wordmask];
 563		w ^= r->pool[(i + tap2) & wordmask];
 564		w ^= r->pool[(i + tap3) & wordmask];
 565		w ^= r->pool[(i + tap4) & wordmask];
 566		w ^= r->pool[(i + tap5) & wordmask];
 567
 568		/* Mix the result back in with a twist */
 569		r->pool[i] = (w >> 3) ^ twist_table[w & 7];
 570
 571		/*
 572		 * Normally, we add 7 bits of rotation to the pool.
 573		 * At the beginning of the pool, add an extra 7 bits
 574		 * rotation, so that successive passes spread the
 575		 * input bits across the pool evenly.
 576		 */
 577		input_rotate = (input_rotate + (i ? 7 : 14)) & 31;
 578	}
 579
 580	r->input_rotate = input_rotate;
 581	r->add_ptr = i;
 582}
 583
 584static void __mix_pool_bytes(struct entropy_store *r, const void *in,
 585			     int nbytes)
 586{
 587	trace_mix_pool_bytes_nolock(r->name, nbytes, _RET_IP_);
 588	_mix_pool_bytes(r, in, nbytes);
 589}
 590
 591static void mix_pool_bytes(struct entropy_store *r, const void *in,
 592			   int nbytes)
 593{
 594	unsigned long flags;
 595
 596	trace_mix_pool_bytes(r->name, nbytes, _RET_IP_);
 597	spin_lock_irqsave(&r->lock, flags);
 598	_mix_pool_bytes(r, in, nbytes);
 599	spin_unlock_irqrestore(&r->lock, flags);
 600}
 601
 602struct fast_pool {
 603	__u32		pool[4];
 604	unsigned long	last;
 605	unsigned short	reg_idx;
 606	unsigned char	count;
 607};
 608
 609/*
 610 * This is a fast mixing routine used by the interrupt randomness
 611 * collector.  It's hardcoded for an 128 bit pool and assumes that any
 612 * locks that might be needed are taken by the caller.
 613 */
 614static void fast_mix(struct fast_pool *f)
 615{
 616	__u32 a = f->pool[0],	b = f->pool[1];
 617	__u32 c = f->pool[2],	d = f->pool[3];
 618
 619	a += b;			c += d;
 620	b = rol32(b, 6);	d = rol32(d, 27);
 621	d ^= a;			b ^= c;
 622
 623	a += b;			c += d;
 624	b = rol32(b, 16);	d = rol32(d, 14);
 625	d ^= a;			b ^= c;
 626
 627	a += b;			c += d;
 628	b = rol32(b, 6);	d = rol32(d, 27);
 629	d ^= a;			b ^= c;
 630
 631	a += b;			c += d;
 632	b = rol32(b, 16);	d = rol32(d, 14);
 633	d ^= a;			b ^= c;
 634
 635	f->pool[0] = a;  f->pool[1] = b;
 636	f->pool[2] = c;  f->pool[3] = d;
 637	f->count++;
 638}
 639
 640static void process_random_ready_list(void)
 641{
 642	unsigned long flags;
 643	struct random_ready_callback *rdy, *tmp;
 644
 645	spin_lock_irqsave(&random_ready_list_lock, flags);
 646	list_for_each_entry_safe(rdy, tmp, &random_ready_list, list) {
 647		struct module *owner = rdy->owner;
 648
 649		list_del_init(&rdy->list);
 650		rdy->func(rdy);
 651		module_put(owner);
 652	}
 653	spin_unlock_irqrestore(&random_ready_list_lock, flags);
 654}
 655
 656/*
 657 * Credit (or debit) the entropy store with n bits of entropy.
 658 * Use credit_entropy_bits_safe() if the value comes from userspace
 659 * or otherwise should be checked for extreme values.
 660 */
 661static void credit_entropy_bits(struct entropy_store *r, int nbits)
 662{
 663	int entropy_count, orig, has_initialized = 0;
 664	const int pool_size = r->poolinfo->poolfracbits;
 665	int nfrac = nbits << ENTROPY_SHIFT;
 666
 667	if (!nbits)
 668		return;
 669
 670retry:
 671	entropy_count = orig = READ_ONCE(r->entropy_count);
 672	if (nfrac < 0) {
 673		/* Debit */
 674		entropy_count += nfrac;
 675	} else {
 676		/*
 677		 * Credit: we have to account for the possibility of
 678		 * overwriting already present entropy.	 Even in the
 679		 * ideal case of pure Shannon entropy, new contributions
 680		 * approach the full value asymptotically:
 681		 *
 682		 * entropy <- entropy + (pool_size - entropy) *
 683		 *	(1 - exp(-add_entropy/pool_size))
 684		 *
 685		 * For add_entropy <= pool_size/2 then
 686		 * (1 - exp(-add_entropy/pool_size)) >=
 687		 *    (add_entropy/pool_size)*0.7869...
 688		 * so we can approximate the exponential with
 689		 * 3/4*add_entropy/pool_size and still be on the
 690		 * safe side by adding at most pool_size/2 at a time.
 691		 *
 692		 * The use of pool_size-2 in the while statement is to
 693		 * prevent rounding artifacts from making the loop
 694		 * arbitrarily long; this limits the loop to log2(pool_size)*2
 695		 * turns no matter how large nbits is.
 696		 */
 697		int pnfrac = nfrac;
 698		const int s = r->poolinfo->poolbitshift + ENTROPY_SHIFT + 2;
 699		/* The +2 corresponds to the /4 in the denominator */
 700
 701		do {
 702			unsigned int anfrac = min(pnfrac, pool_size/2);
 703			unsigned int add =
 704				((pool_size - entropy_count)*anfrac*3) >> s;
 705
 706			entropy_count += add;
 707			pnfrac -= anfrac;
 708		} while (unlikely(entropy_count < pool_size-2 && pnfrac));
 709	}
 710
 711	if (WARN_ON(entropy_count < 0)) {
 712		pr_warn("negative entropy/overflow: pool %s count %d\n",
 713			r->name, entropy_count);
 714		entropy_count = 0;
 715	} else if (entropy_count > pool_size)
 716		entropy_count = pool_size;
 717	if (cmpxchg(&r->entropy_count, orig, entropy_count) != orig)
 718		goto retry;
 719
 720	if (has_initialized) {
 721		r->initialized = 1;
 722		kill_fasync(&fasync, SIGIO, POLL_IN);
 723	}
 724
 725	trace_credit_entropy_bits(r->name, nbits,
 726				  entropy_count >> ENTROPY_SHIFT, _RET_IP_);
 727
 728	if (r == &input_pool) {
 729		int entropy_bits = entropy_count >> ENTROPY_SHIFT;
 730
 731		if (crng_init < 2) {
 732			if (entropy_bits < 128)
 733				return;
 734			crng_reseed(&primary_crng, r);
 735			entropy_bits = ENTROPY_BITS(r);
 736		}
 737	}
 738}
 739
 740static int credit_entropy_bits_safe(struct entropy_store *r, int nbits)
 741{
 742	const int nbits_max = r->poolinfo->poolwords * 32;
 743
 744	if (nbits < 0)
 745		return -EINVAL;
 746
 747	/* Cap the value to avoid overflows */
 748	nbits = min(nbits,  nbits_max);
 749
 750	credit_entropy_bits(r, nbits);
 751	return 0;
 752}
 753
 754/*********************************************************************
 755 *
 756 * CRNG using CHACHA20
 757 *
 758 *********************************************************************/
 759
 760#define CRNG_RESEED_INTERVAL (300*HZ)
 761
 762static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait);
 763
 764#ifdef CONFIG_NUMA
 765/*
 766 * Hack to deal with crazy userspace progams when they are all trying
 767 * to access /dev/urandom in parallel.  The programs are almost
 768 * certainly doing something terribly wrong, but we'll work around
 769 * their brain damage.
 770 */
 771static struct crng_state **crng_node_pool __read_mostly;
 772#endif
 773
 774static void invalidate_batched_entropy(void);
 775static void numa_crng_init(void);
 776
 777static bool trust_cpu __ro_after_init = IS_ENABLED(CONFIG_RANDOM_TRUST_CPU);
 778static int __init parse_trust_cpu(char *arg)
 779{
 780	return kstrtobool(arg, &trust_cpu);
 781}
 782early_param("random.trust_cpu", parse_trust_cpu);
 783
 784static bool crng_init_try_arch(struct crng_state *crng)
 785{
 786	int		i;
 787	bool		arch_init = true;
 788	unsigned long	rv;
 789
 790	for (i = 4; i < 16; i++) {
 791		if (!arch_get_random_seed_long(&rv) &&
 792		    !arch_get_random_long(&rv)) {
 793			rv = random_get_entropy();
 794			arch_init = false;
 795		}
 796		crng->state[i] ^= rv;
 797	}
 798
 799	return arch_init;
 800}
 801
 802static bool __init crng_init_try_arch_early(struct crng_state *crng)
 803{
 804	int		i;
 805	bool		arch_init = true;
 806	unsigned long	rv;
 807
 808	for (i = 4; i < 16; i++) {
 809		if (!arch_get_random_seed_long_early(&rv) &&
 810		    !arch_get_random_long_early(&rv)) {
 811			rv = random_get_entropy();
 812			arch_init = false;
 813		}
 814		crng->state[i] ^= rv;
 815	}
 816
 817	return arch_init;
 818}
 819
 820static void __maybe_unused crng_initialize_secondary(struct crng_state *crng)
 821{
 822	memcpy(&crng->state[0], "expand 32-byte k", 16);
 823	_get_random_bytes(&crng->state[4], sizeof(__u32) * 12);
 824	crng_init_try_arch(crng);
 825	crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1;
 826}
 827
 828static void __init crng_initialize_primary(struct crng_state *crng)
 829{
 830	memcpy(&crng->state[0], "expand 32-byte k", 16);
 831	_extract_entropy(&input_pool, &crng->state[4], sizeof(__u32) * 12, 0);
 832	if (crng_init_try_arch_early(crng) && trust_cpu) {
 833		invalidate_batched_entropy();
 834		numa_crng_init();
 835		crng_init = 2;
 836		pr_notice("crng done (trusting CPU's manufacturer)\n");
 837	}
 838	crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1;
 839}
 840
 841#ifdef CONFIG_NUMA
 842static void do_numa_crng_init(struct work_struct *work)
 843{
 844	int i;
 845	struct crng_state *crng;
 846	struct crng_state **pool;
 847
 848	pool = kcalloc(nr_node_ids, sizeof(*pool), GFP_KERNEL|__GFP_NOFAIL);
 849	for_each_online_node(i) {
 850		crng = kmalloc_node(sizeof(struct crng_state),
 851				    GFP_KERNEL | __GFP_NOFAIL, i);
 852		spin_lock_init(&crng->lock);
 853		crng_initialize_secondary(crng);
 854		pool[i] = crng;
 855	}
 856	mb();
 857	if (cmpxchg(&crng_node_pool, NULL, pool)) {
 858		for_each_node(i)
 859			kfree(pool[i]);
 860		kfree(pool);
 861	}
 862}
 863
 864static DECLARE_WORK(numa_crng_init_work, do_numa_crng_init);
 865
 866static void numa_crng_init(void)
 867{
 868	schedule_work(&numa_crng_init_work);
 869}
 870#else
 871static void numa_crng_init(void) {}
 872#endif
 873
 874/*
 875 * crng_fast_load() can be called by code in the interrupt service
 876 * path.  So we can't afford to dilly-dally.
 877 */
 878static int crng_fast_load(const char *cp, size_t len)
 879{
 880	unsigned long flags;
 881	char *p;
 882
 883	if (!spin_trylock_irqsave(&primary_crng.lock, flags))
 884		return 0;
 885	if (crng_init != 0) {
 886		spin_unlock_irqrestore(&primary_crng.lock, flags);
 887		return 0;
 888	}
 889	p = (unsigned char *) &primary_crng.state[4];
 890	while (len > 0 && crng_init_cnt < CRNG_INIT_CNT_THRESH) {
 891		p[crng_init_cnt % CHACHA_KEY_SIZE] ^= *cp;
 892		cp++; crng_init_cnt++; len--;
 893	}
 894	spin_unlock_irqrestore(&primary_crng.lock, flags);
 895	if (crng_init_cnt >= CRNG_INIT_CNT_THRESH) {
 896		invalidate_batched_entropy();
 897		crng_init = 1;
 898		pr_notice("fast init done\n");
 899	}
 900	return 1;
 901}
 902
 903/*
 904 * crng_slow_load() is called by add_device_randomness, which has two
 905 * attributes.  (1) We can't trust the buffer passed to it is
 906 * guaranteed to be unpredictable (so it might not have any entropy at
 907 * all), and (2) it doesn't have the performance constraints of
 908 * crng_fast_load().
 909 *
 910 * So we do something more comprehensive which is guaranteed to touch
 911 * all of the primary_crng's state, and which uses a LFSR with a
 912 * period of 255 as part of the mixing algorithm.  Finally, we do
 913 * *not* advance crng_init_cnt since buffer we may get may be something
 914 * like a fixed DMI table (for example), which might very well be
 915 * unique to the machine, but is otherwise unvarying.
 916 */
 917static int crng_slow_load(const char *cp, size_t len)
 918{
 919	unsigned long		flags;
 920	static unsigned char	lfsr = 1;
 921	unsigned char		tmp;
 922	unsigned		i, max = CHACHA_KEY_SIZE;
 923	const char *		src_buf = cp;
 924	char *			dest_buf = (char *) &primary_crng.state[4];
 925
 926	if (!spin_trylock_irqsave(&primary_crng.lock, flags))
 927		return 0;
 928	if (crng_init != 0) {
 929		spin_unlock_irqrestore(&primary_crng.lock, flags);
 930		return 0;
 931	}
 932	if (len > max)
 933		max = len;
 934
 935	for (i = 0; i < max ; i++) {
 936		tmp = lfsr;
 937		lfsr >>= 1;
 938		if (tmp & 1)
 939			lfsr ^= 0xE1;
 940		tmp = dest_buf[i % CHACHA_KEY_SIZE];
 941		dest_buf[i % CHACHA_KEY_SIZE] ^= src_buf[i % len] ^ lfsr;
 942		lfsr += (tmp << 3) | (tmp >> 5);
 943	}
 944	spin_unlock_irqrestore(&primary_crng.lock, flags);
 945	return 1;
 946}
 947
 948static void crng_reseed(struct crng_state *crng, struct entropy_store *r)
 949{
 950	unsigned long	flags;
 951	int		i, num;
 952	union {
 953		__u8	block[CHACHA_BLOCK_SIZE];
 954		__u32	key[8];
 955	} buf;
 956
 957	if (r) {
 958		num = extract_entropy(r, &buf, 32, 16, 0);
 959		if (num == 0)
 960			return;
 961	} else {
 962		_extract_crng(&primary_crng, buf.block);
 963		_crng_backtrack_protect(&primary_crng, buf.block,
 964					CHACHA_KEY_SIZE);
 965	}
 966	spin_lock_irqsave(&crng->lock, flags);
 967	for (i = 0; i < 8; i++) {
 968		unsigned long	rv;
 969		if (!arch_get_random_seed_long(&rv) &&
 970		    !arch_get_random_long(&rv))
 971			rv = random_get_entropy();
 972		crng->state[i+4] ^= buf.key[i] ^ rv;
 973	}
 974	memzero_explicit(&buf, sizeof(buf));
 975	crng->init_time = jiffies;
 976	spin_unlock_irqrestore(&crng->lock, flags);
 977	if (crng == &primary_crng && crng_init < 2) {
 978		invalidate_batched_entropy();
 979		numa_crng_init();
 980		crng_init = 2;
 981		process_random_ready_list();
 982		wake_up_interruptible(&crng_init_wait);
 983		kill_fasync(&fasync, SIGIO, POLL_IN);
 984		pr_notice("crng init done\n");
 985		if (unseeded_warning.missed) {
 986			pr_notice("%d get_random_xx warning(s) missed due to ratelimiting\n",
 987				  unseeded_warning.missed);
 988			unseeded_warning.missed = 0;
 989		}
 990		if (urandom_warning.missed) {
 991			pr_notice("%d urandom warning(s) missed due to ratelimiting\n",
 992				  urandom_warning.missed);
 993			urandom_warning.missed = 0;
 994		}
 995	}
 996}
 997
 998static void _extract_crng(struct crng_state *crng,
 999			  __u8 out[CHACHA_BLOCK_SIZE])
1000{
1001	unsigned long v, flags;
1002
1003	if (crng_ready() &&
1004	    (time_after(crng_global_init_time, crng->init_time) ||
1005	     time_after(jiffies, crng->init_time + CRNG_RESEED_INTERVAL)))
1006		crng_reseed(crng, crng == &primary_crng ? &input_pool : NULL);
1007	spin_lock_irqsave(&crng->lock, flags);
1008	if (arch_get_random_long(&v))
1009		crng->state[14] ^= v;
1010	chacha20_block(&crng->state[0], out);
1011	if (crng->state[12] == 0)
1012		crng->state[13]++;
1013	spin_unlock_irqrestore(&crng->lock, flags);
1014}
1015
1016static void extract_crng(__u8 out[CHACHA_BLOCK_SIZE])
1017{
1018	struct crng_state *crng = NULL;
1019
1020#ifdef CONFIG_NUMA
1021	if (crng_node_pool)
1022		crng = crng_node_pool[numa_node_id()];
1023	if (crng == NULL)
1024#endif
1025		crng = &primary_crng;
1026	_extract_crng(crng, out);
1027}
1028
1029/*
1030 * Use the leftover bytes from the CRNG block output (if there is
1031 * enough) to mutate the CRNG key to provide backtracking protection.
1032 */
1033static void _crng_backtrack_protect(struct crng_state *crng,
1034				    __u8 tmp[CHACHA_BLOCK_SIZE], int used)
1035{
1036	unsigned long	flags;
1037	__u32		*s, *d;
1038	int		i;
1039
1040	used = round_up(used, sizeof(__u32));
1041	if (used + CHACHA_KEY_SIZE > CHACHA_BLOCK_SIZE) {
1042		extract_crng(tmp);
1043		used = 0;
1044	}
1045	spin_lock_irqsave(&crng->lock, flags);
1046	s = (__u32 *) &tmp[used];
1047	d = &crng->state[4];
1048	for (i=0; i < 8; i++)
1049		*d++ ^= *s++;
1050	spin_unlock_irqrestore(&crng->lock, flags);
1051}
1052
1053static void crng_backtrack_protect(__u8 tmp[CHACHA_BLOCK_SIZE], int used)
1054{
1055	struct crng_state *crng = NULL;
1056
1057#ifdef CONFIG_NUMA
1058	if (crng_node_pool)
1059		crng = crng_node_pool[numa_node_id()];
1060	if (crng == NULL)
1061#endif
1062		crng = &primary_crng;
1063	_crng_backtrack_protect(crng, tmp, used);
1064}
1065
1066static ssize_t extract_crng_user(void __user *buf, size_t nbytes)
1067{
1068	ssize_t ret = 0, i = CHACHA_BLOCK_SIZE;
1069	__u8 tmp[CHACHA_BLOCK_SIZE] __aligned(4);
1070	int large_request = (nbytes > 256);
1071
1072	while (nbytes) {
1073		if (large_request && need_resched()) {
1074			if (signal_pending(current)) {
1075				if (ret == 0)
1076					ret = -ERESTARTSYS;
1077				break;
1078			}
1079			schedule();
1080		}
1081
1082		extract_crng(tmp);
1083		i = min_t(int, nbytes, CHACHA_BLOCK_SIZE);
1084		if (copy_to_user(buf, tmp, i)) {
1085			ret = -EFAULT;
1086			break;
1087		}
1088
1089		nbytes -= i;
1090		buf += i;
1091		ret += i;
1092	}
1093	crng_backtrack_protect(tmp, i);
1094
1095	/* Wipe data just written to memory */
1096	memzero_explicit(tmp, sizeof(tmp));
1097
1098	return ret;
1099}
1100
1101
1102/*********************************************************************
1103 *
1104 * Entropy input management
1105 *
1106 *********************************************************************/
1107
1108/* There is one of these per entropy source */
1109struct timer_rand_state {
1110	cycles_t last_time;
1111	long last_delta, last_delta2;
1112};
1113
1114#define INIT_TIMER_RAND_STATE { INITIAL_JIFFIES, };
1115
1116/*
1117 * Add device- or boot-specific data to the input pool to help
1118 * initialize it.
1119 *
1120 * None of this adds any entropy; it is meant to avoid the problem of
1121 * the entropy pool having similar initial state across largely
1122 * identical devices.
1123 */
1124void add_device_randomness(const void *buf, unsigned int size)
1125{
1126	unsigned long time = random_get_entropy() ^ jiffies;
1127	unsigned long flags;
1128
1129	if (!crng_ready() && size)
1130		crng_slow_load(buf, size);
1131
1132	trace_add_device_randomness(size, _RET_IP_);
1133	spin_lock_irqsave(&input_pool.lock, flags);
1134	_mix_pool_bytes(&input_pool, buf, size);
1135	_mix_pool_bytes(&input_pool, &time, sizeof(time));
1136	spin_unlock_irqrestore(&input_pool.lock, flags);
1137}
1138EXPORT_SYMBOL(add_device_randomness);
1139
1140static struct timer_rand_state input_timer_state = INIT_TIMER_RAND_STATE;
1141
1142/*
1143 * This function adds entropy to the entropy "pool" by using timing
1144 * delays.  It uses the timer_rand_state structure to make an estimate
1145 * of how many bits of entropy this call has added to the pool.
1146 *
1147 * The number "num" is also added to the pool - it should somehow describe
1148 * the type of event which just happened.  This is currently 0-255 for
1149 * keyboard scan codes, and 256 upwards for interrupts.
1150 *
1151 */
1152static void add_timer_randomness(struct timer_rand_state *state, unsigned num)
1153{
1154	struct entropy_store	*r;
1155	struct {
1156		long jiffies;
1157		unsigned cycles;
1158		unsigned num;
1159	} sample;
1160	long delta, delta2, delta3;
1161
1162	sample.jiffies = jiffies;
1163	sample.cycles = random_get_entropy();
1164	sample.num = num;
1165	r = &input_pool;
1166	mix_pool_bytes(r, &sample, sizeof(sample));
1167
1168	/*
1169	 * Calculate number of bits of randomness we probably added.
1170	 * We take into account the first, second and third-order deltas
1171	 * in order to make our estimate.
1172	 */
1173	delta = sample.jiffies - READ_ONCE(state->last_time);
1174	WRITE_ONCE(state->last_time, sample.jiffies);
1175
1176	delta2 = delta - READ_ONCE(state->last_delta);
1177	WRITE_ONCE(state->last_delta, delta);
1178
1179	delta3 = delta2 - READ_ONCE(state->last_delta2);
1180	WRITE_ONCE(state->last_delta2, delta2);
1181
1182	if (delta < 0)
1183		delta = -delta;
1184	if (delta2 < 0)
1185		delta2 = -delta2;
1186	if (delta3 < 0)
1187		delta3 = -delta3;
1188	if (delta > delta2)
1189		delta = delta2;
1190	if (delta > delta3)
1191		delta = delta3;
1192
1193	/*
1194	 * delta is now minimum absolute delta.
1195	 * Round down by 1 bit on general principles,
1196	 * and limit entropy estimate to 12 bits.
1197	 */
1198	credit_entropy_bits(r, min_t(int, fls(delta>>1), 11));
1199}
1200
1201void add_input_randomness(unsigned int type, unsigned int code,
1202				 unsigned int value)
1203{
1204	static unsigned char last_value;
1205
1206	/* ignore autorepeat and the like */
1207	if (value == last_value)
1208		return;
1209
1210	last_value = value;
1211	add_timer_randomness(&input_timer_state,
1212			     (type << 4) ^ code ^ (code >> 4) ^ value);
1213	trace_add_input_randomness(ENTROPY_BITS(&input_pool));
1214}
1215EXPORT_SYMBOL_GPL(add_input_randomness);
1216
1217static DEFINE_PER_CPU(struct fast_pool, irq_randomness);
1218
1219#ifdef ADD_INTERRUPT_BENCH
1220static unsigned long avg_cycles, avg_deviation;
1221
1222#define AVG_SHIFT 8     /* Exponential average factor k=1/256 */
1223#define FIXED_1_2 (1 << (AVG_SHIFT-1))
1224
1225static void add_interrupt_bench(cycles_t start)
1226{
1227        long delta = random_get_entropy() - start;
1228
1229        /* Use a weighted moving average */
1230        delta = delta - ((avg_cycles + FIXED_1_2) >> AVG_SHIFT);
1231        avg_cycles += delta;
1232        /* And average deviation */
1233        delta = abs(delta) - ((avg_deviation + FIXED_1_2) >> AVG_SHIFT);
1234        avg_deviation += delta;
1235}
1236#else
1237#define add_interrupt_bench(x)
1238#endif
1239
1240static __u32 get_reg(struct fast_pool *f, struct pt_regs *regs)
1241{
1242	__u32 *ptr = (__u32 *) regs;
1243	unsigned int idx;
1244
1245	if (regs == NULL)
1246		return 0;
1247	idx = READ_ONCE(f->reg_idx);
1248	if (idx >= sizeof(struct pt_regs) / sizeof(__u32))
1249		idx = 0;
1250	ptr += idx++;
1251	WRITE_ONCE(f->reg_idx, idx);
1252	return *ptr;
1253}
1254
1255void add_interrupt_randomness(int irq, int irq_flags)
1256{
1257	struct entropy_store	*r;
1258	struct fast_pool	*fast_pool = this_cpu_ptr(&irq_randomness);
1259	struct pt_regs		*regs = get_irq_regs();
1260	unsigned long		now = jiffies;
1261	cycles_t		cycles = random_get_entropy();
1262	__u32			c_high, j_high;
1263	__u64			ip;
1264	unsigned long		seed;
1265	int			credit = 0;
1266
1267	if (cycles == 0)
1268		cycles = get_reg(fast_pool, regs);
1269	c_high = (sizeof(cycles) > 4) ? cycles >> 32 : 0;
1270	j_high = (sizeof(now) > 4) ? now >> 32 : 0;
1271	fast_pool->pool[0] ^= cycles ^ j_high ^ irq;
1272	fast_pool->pool[1] ^= now ^ c_high;
1273	ip = regs ? instruction_pointer(regs) : _RET_IP_;
1274	fast_pool->pool[2] ^= ip;
1275	fast_pool->pool[3] ^= (sizeof(ip) > 4) ? ip >> 32 :
1276		get_reg(fast_pool, regs);
1277
1278	fast_mix(fast_pool);
1279	add_interrupt_bench(cycles);
1280	this_cpu_add(net_rand_state.s1, fast_pool->pool[cycles & 3]);
1281
1282	if (unlikely(crng_init == 0)) {
1283		if ((fast_pool->count >= 64) &&
1284		    crng_fast_load((char *) fast_pool->pool,
1285				   sizeof(fast_pool->pool))) {
1286			fast_pool->count = 0;
1287			fast_pool->last = now;
1288		}
1289		return;
1290	}
1291
1292	if ((fast_pool->count < 64) &&
1293	    !time_after(now, fast_pool->last + HZ))
1294		return;
1295
1296	r = &input_pool;
1297	if (!spin_trylock(&r->lock))
1298		return;
1299
1300	fast_pool->last = now;
1301	__mix_pool_bytes(r, &fast_pool->pool, sizeof(fast_pool->pool));
1302
1303	/*
1304	 * If we have architectural seed generator, produce a seed and
1305	 * add it to the pool.  For the sake of paranoia don't let the
1306	 * architectural seed generator dominate the input from the
1307	 * interrupt noise.
1308	 */
1309	if (arch_get_random_seed_long(&seed)) {
1310		__mix_pool_bytes(r, &seed, sizeof(seed));
1311		credit = 1;
1312	}
1313	spin_unlock(&r->lock);
1314
1315	fast_pool->count = 0;
1316
1317	/* award one bit for the contents of the fast pool */
1318	credit_entropy_bits(r, credit + 1);
1319}
1320EXPORT_SYMBOL_GPL(add_interrupt_randomness);
1321
1322#ifdef CONFIG_BLOCK
1323void add_disk_randomness(struct gendisk *disk)
1324{
1325	if (!disk || !disk->random)
1326		return;
1327	/* first major is 1, so we get >= 0x200 here */
1328	add_timer_randomness(disk->random, 0x100 + disk_devt(disk));
1329	trace_add_disk_randomness(disk_devt(disk), ENTROPY_BITS(&input_pool));
1330}
1331EXPORT_SYMBOL_GPL(add_disk_randomness);
1332#endif
1333
1334/*********************************************************************
1335 *
1336 * Entropy extraction routines
1337 *
1338 *********************************************************************/
1339
1340/*
1341 * This function decides how many bytes to actually take from the
1342 * given pool, and also debits the entropy count accordingly.
1343 */
1344static size_t account(struct entropy_store *r, size_t nbytes, int min,
1345		      int reserved)
1346{
1347	int entropy_count, orig, have_bytes;
1348	size_t ibytes, nfrac;
1349
1350	BUG_ON(r->entropy_count > r->poolinfo->poolfracbits);
1351
1352	/* Can we pull enough? */
1353retry:
1354	entropy_count = orig = READ_ONCE(r->entropy_count);
1355	ibytes = nbytes;
1356	/* never pull more than available */
1357	have_bytes = entropy_count >> (ENTROPY_SHIFT + 3);
1358
1359	if ((have_bytes -= reserved) < 0)
1360		have_bytes = 0;
1361	ibytes = min_t(size_t, ibytes, have_bytes);
1362	if (ibytes < min)
1363		ibytes = 0;
1364
1365	if (WARN_ON(entropy_count < 0)) {
1366		pr_warn("negative entropy count: pool %s count %d\n",
1367			r->name, entropy_count);
1368		entropy_count = 0;
1369	}
1370	nfrac = ibytes << (ENTROPY_SHIFT + 3);
1371	if ((size_t) entropy_count > nfrac)
1372		entropy_count -= nfrac;
1373	else
1374		entropy_count = 0;
1375
1376	if (cmpxchg(&r->entropy_count, orig, entropy_count) != orig)
1377		goto retry;
1378
1379	trace_debit_entropy(r->name, 8 * ibytes);
1380	if (ibytes && ENTROPY_BITS(r) < random_write_wakeup_bits) {
1381		wake_up_interruptible(&random_write_wait);
1382		kill_fasync(&fasync, SIGIO, POLL_OUT);
1383	}
1384
1385	return ibytes;
1386}
1387
1388/*
1389 * This function does the actual extraction for extract_entropy and
1390 * extract_entropy_user.
1391 *
1392 * Note: we assume that .poolwords is a multiple of 16 words.
1393 */
1394static void extract_buf(struct entropy_store *r, __u8 *out)
1395{
1396	int i;
1397	union {
1398		__u32 w[5];
1399		unsigned long l[LONGS(20)];
1400	} hash;
1401	__u32 workspace[SHA1_WORKSPACE_WORDS];
1402	unsigned long flags;
1403
1404	/*
1405	 * If we have an architectural hardware random number
1406	 * generator, use it for SHA's initial vector
1407	 */
1408	sha1_init(hash.w);
1409	for (i = 0; i < LONGS(20); i++) {
1410		unsigned long v;
1411		if (!arch_get_random_long(&v))
1412			break;
1413		hash.l[i] = v;
1414	}
1415
1416	/* Generate a hash across the pool, 16 words (512 bits) at a time */
1417	spin_lock_irqsave(&r->lock, flags);
1418	for (i = 0; i < r->poolinfo->poolwords; i += 16)
1419		sha1_transform(hash.w, (__u8 *)(r->pool + i), workspace);
1420
1421	/*
1422	 * We mix the hash back into the pool to prevent backtracking
1423	 * attacks (where the attacker knows the state of the pool
1424	 * plus the current outputs, and attempts to find previous
1425	 * ouputs), unless the hash function can be inverted. By
1426	 * mixing at least a SHA1 worth of hash data back, we make
1427	 * brute-forcing the feedback as hard as brute-forcing the
1428	 * hash.
1429	 */
1430	__mix_pool_bytes(r, hash.w, sizeof(hash.w));
1431	spin_unlock_irqrestore(&r->lock, flags);
1432
1433	memzero_explicit(workspace, sizeof(workspace));
1434
1435	/*
1436	 * In case the hash function has some recognizable output
1437	 * pattern, we fold it in half. Thus, we always feed back
1438	 * twice as much data as we output.
1439	 */
1440	hash.w[0] ^= hash.w[3];
1441	hash.w[1] ^= hash.w[4];
1442	hash.w[2] ^= rol32(hash.w[2], 16);
1443
1444	memcpy(out, &hash, EXTRACT_SIZE);
1445	memzero_explicit(&hash, sizeof(hash));
1446}
1447
1448static ssize_t _extract_entropy(struct entropy_store *r, void *buf,
1449				size_t nbytes, int fips)
1450{
1451	ssize_t ret = 0, i;
1452	__u8 tmp[EXTRACT_SIZE];
1453	unsigned long flags;
1454
1455	while (nbytes) {
1456		extract_buf(r, tmp);
1457
1458		if (fips) {
1459			spin_lock_irqsave(&r->lock, flags);
1460			if (!memcmp(tmp, r->last_data, EXTRACT_SIZE))
1461				panic("Hardware RNG duplicated output!\n");
1462			memcpy(r->last_data, tmp, EXTRACT_SIZE);
1463			spin_unlock_irqrestore(&r->lock, flags);
1464		}
1465		i = min_t(int, nbytes, EXTRACT_SIZE);
1466		memcpy(buf, tmp, i);
1467		nbytes -= i;
1468		buf += i;
1469		ret += i;
1470	}
1471
1472	/* Wipe data just returned from memory */
1473	memzero_explicit(tmp, sizeof(tmp));
1474
1475	return ret;
1476}
1477
1478/*
1479 * This function extracts randomness from the "entropy pool", and
1480 * returns it in a buffer.
1481 *
1482 * The min parameter specifies the minimum amount we can pull before
1483 * failing to avoid races that defeat catastrophic reseeding while the
1484 * reserved parameter indicates how much entropy we must leave in the
1485 * pool after each pull to avoid starving other readers.
1486 */
1487static ssize_t extract_entropy(struct entropy_store *r, void *buf,
1488				 size_t nbytes, int min, int reserved)
1489{
1490	__u8 tmp[EXTRACT_SIZE];
1491	unsigned long flags;
1492
1493	/* if last_data isn't primed, we need EXTRACT_SIZE extra bytes */
1494	if (fips_enabled) {
1495		spin_lock_irqsave(&r->lock, flags);
1496		if (!r->last_data_init) {
1497			r->last_data_init = 1;
1498			spin_unlock_irqrestore(&r->lock, flags);
1499			trace_extract_entropy(r->name, EXTRACT_SIZE,
1500					      ENTROPY_BITS(r), _RET_IP_);
1501			extract_buf(r, tmp);
1502			spin_lock_irqsave(&r->lock, flags);
1503			memcpy(r->last_data, tmp, EXTRACT_SIZE);
1504		}
1505		spin_unlock_irqrestore(&r->lock, flags);
1506	}
1507
1508	trace_extract_entropy(r->name, nbytes, ENTROPY_BITS(r), _RET_IP_);
1509	nbytes = account(r, nbytes, min, reserved);
1510
1511	return _extract_entropy(r, buf, nbytes, fips_enabled);
1512}
1513
1514#define warn_unseeded_randomness(previous) \
1515	_warn_unseeded_randomness(__func__, (void *) _RET_IP_, (previous))
1516
1517static void _warn_unseeded_randomness(const char *func_name, void *caller,
1518				      void **previous)
1519{
1520#ifdef CONFIG_WARN_ALL_UNSEEDED_RANDOM
1521	const bool print_once = false;
1522#else
1523	static bool print_once __read_mostly;
1524#endif
1525
1526	if (print_once ||
1527	    crng_ready() ||
1528	    (previous && (caller == READ_ONCE(*previous))))
1529		return;
1530	WRITE_ONCE(*previous, caller);
1531#ifndef CONFIG_WARN_ALL_UNSEEDED_RANDOM
1532	print_once = true;
1533#endif
1534	if (__ratelimit(&unseeded_warning))
1535		printk_deferred(KERN_NOTICE "random: %s called from %pS "
1536				"with crng_init=%d\n", func_name, caller,
1537				crng_init);
1538}
1539
1540/*
1541 * This function is the exported kernel interface.  It returns some
1542 * number of good random numbers, suitable for key generation, seeding
1543 * TCP sequence numbers, etc.  It does not rely on the hardware random
1544 * number generator.  For random bytes direct from the hardware RNG
1545 * (when available), use get_random_bytes_arch(). In order to ensure
1546 * that the randomness provided by this function is okay, the function
1547 * wait_for_random_bytes() should be called and return 0 at least once
1548 * at any point prior.
1549 */
1550static void _get_random_bytes(void *buf, int nbytes)
1551{
1552	__u8 tmp[CHACHA_BLOCK_SIZE] __aligned(4);
1553
1554	trace_get_random_bytes(nbytes, _RET_IP_);
1555
1556	while (nbytes >= CHACHA_BLOCK_SIZE) {
1557		extract_crng(buf);
1558		buf += CHACHA_BLOCK_SIZE;
1559		nbytes -= CHACHA_BLOCK_SIZE;
1560	}
1561
1562	if (nbytes > 0) {
1563		extract_crng(tmp);
1564		memcpy(buf, tmp, nbytes);
1565		crng_backtrack_protect(tmp, nbytes);
1566	} else
1567		crng_backtrack_protect(tmp, CHACHA_BLOCK_SIZE);
1568	memzero_explicit(tmp, sizeof(tmp));
1569}
1570
1571void get_random_bytes(void *buf, int nbytes)
1572{
1573	static void *previous;
1574
1575	warn_unseeded_randomness(&previous);
1576	_get_random_bytes(buf, nbytes);
1577}
1578EXPORT_SYMBOL(get_random_bytes);
1579
1580
1581/*
1582 * Each time the timer fires, we expect that we got an unpredictable
1583 * jump in the cycle counter. Even if the timer is running on another
1584 * CPU, the timer activity will be touching the stack of the CPU that is
1585 * generating entropy..
1586 *
1587 * Note that we don't re-arm the timer in the timer itself - we are
1588 * happy to be scheduled away, since that just makes the load more
1589 * complex, but we do not want the timer to keep ticking unless the
1590 * entropy loop is running.
1591 *
1592 * So the re-arming always happens in the entropy loop itself.
1593 */
1594static void entropy_timer(struct timer_list *t)
1595{
1596	credit_entropy_bits(&input_pool, 1);
1597}
1598
1599/*
1600 * If we have an actual cycle counter, see if we can
1601 * generate enough entropy with timing noise
1602 */
1603static void try_to_generate_entropy(void)
1604{
1605	struct {
1606		unsigned long now;
1607		struct timer_list timer;
1608	} stack;
1609
1610	stack.now = random_get_entropy();
1611
1612	/* Slow counter - or none. Don't even bother */
1613	if (stack.now == random_get_entropy())
1614		return;
1615
1616	timer_setup_on_stack(&stack.timer, entropy_timer, 0);
1617	while (!crng_ready()) {
1618		if (!timer_pending(&stack.timer))
1619			mod_timer(&stack.timer, jiffies+1);
1620		mix_pool_bytes(&input_pool, &stack.now, sizeof(stack.now));
1621		schedule();
1622		stack.now = random_get_entropy();
1623	}
1624
1625	del_timer_sync(&stack.timer);
1626	destroy_timer_on_stack(&stack.timer);
1627	mix_pool_bytes(&input_pool, &stack.now, sizeof(stack.now));
1628}
1629
1630/*
1631 * Wait for the urandom pool to be seeded and thus guaranteed to supply
1632 * cryptographically secure random numbers. This applies to: the /dev/urandom
1633 * device, the get_random_bytes function, and the get_random_{u32,u64,int,long}
1634 * family of functions. Using any of these functions without first calling
1635 * this function forfeits the guarantee of security.
1636 *
1637 * Returns: 0 if the urandom pool has been seeded.
1638 *          -ERESTARTSYS if the function was interrupted by a signal.
1639 */
1640int wait_for_random_bytes(void)
1641{
1642	if (likely(crng_ready()))
1643		return 0;
1644
1645	do {
1646		int ret;
1647		ret = wait_event_interruptible_timeout(crng_init_wait, crng_ready(), HZ);
1648		if (ret)
1649			return ret > 0 ? 0 : ret;
1650
1651		try_to_generate_entropy();
1652	} while (!crng_ready());
1653
1654	return 0;
1655}
1656EXPORT_SYMBOL(wait_for_random_bytes);
1657
1658/*
1659 * Returns whether or not the urandom pool has been seeded and thus guaranteed
1660 * to supply cryptographically secure random numbers. This applies to: the
1661 * /dev/urandom device, the get_random_bytes function, and the get_random_{u32,
1662 * ,u64,int,long} family of functions.
1663 *
1664 * Returns: true if the urandom pool has been seeded.
1665 *          false if the urandom pool has not been seeded.
1666 */
1667bool rng_is_initialized(void)
1668{
1669	return crng_ready();
1670}
1671EXPORT_SYMBOL(rng_is_initialized);
1672
1673/*
1674 * Add a callback function that will be invoked when the nonblocking
1675 * pool is initialised.
1676 *
1677 * returns: 0 if callback is successfully added
1678 *	    -EALREADY if pool is already initialised (callback not called)
1679 *	    -ENOENT if module for callback is not alive
1680 */
1681int add_random_ready_callback(struct random_ready_callback *rdy)
1682{
1683	struct module *owner;
1684	unsigned long flags;
1685	int err = -EALREADY;
1686
1687	if (crng_ready())
1688		return err;
1689
1690	owner = rdy->owner;
1691	if (!try_module_get(owner))
1692		return -ENOENT;
1693
1694	spin_lock_irqsave(&random_ready_list_lock, flags);
1695	if (crng_ready())
1696		goto out;
1697
1698	owner = NULL;
1699
1700	list_add(&rdy->list, &random_ready_list);
1701	err = 0;
1702
1703out:
1704	spin_unlock_irqrestore(&random_ready_list_lock, flags);
1705
1706	module_put(owner);
1707
1708	return err;
1709}
1710EXPORT_SYMBOL(add_random_ready_callback);
1711
1712/*
1713 * Delete a previously registered readiness callback function.
1714 */
1715void del_random_ready_callback(struct random_ready_callback *rdy)
1716{
1717	unsigned long flags;
1718	struct module *owner = NULL;
1719
1720	spin_lock_irqsave(&random_ready_list_lock, flags);
1721	if (!list_empty(&rdy->list)) {
1722		list_del_init(&rdy->list);
1723		owner = rdy->owner;
1724	}
1725	spin_unlock_irqrestore(&random_ready_list_lock, flags);
1726
1727	module_put(owner);
1728}
1729EXPORT_SYMBOL(del_random_ready_callback);
1730
1731/*
1732 * This function will use the architecture-specific hardware random
1733 * number generator if it is available.  The arch-specific hw RNG will
1734 * almost certainly be faster than what we can do in software, but it
1735 * is impossible to verify that it is implemented securely (as
1736 * opposed, to, say, the AES encryption of a sequence number using a
1737 * key known by the NSA).  So it's useful if we need the speed, but
1738 * only if we're willing to trust the hardware manufacturer not to
1739 * have put in a back door.
1740 *
1741 * Return number of bytes filled in.
1742 */
1743int __must_check get_random_bytes_arch(void *buf, int nbytes)
1744{
1745	int left = nbytes;
1746	char *p = buf;
1747
1748	trace_get_random_bytes_arch(left, _RET_IP_);
1749	while (left) {
1750		unsigned long v;
1751		int chunk = min_t(int, left, sizeof(unsigned long));
1752
1753		if (!arch_get_random_long(&v))
1754			break;
1755
1756		memcpy(p, &v, chunk);
1757		p += chunk;
1758		left -= chunk;
1759	}
1760
1761	return nbytes - left;
1762}
1763EXPORT_SYMBOL(get_random_bytes_arch);
1764
1765/*
1766 * init_std_data - initialize pool with system data
1767 *
1768 * @r: pool to initialize
1769 *
1770 * This function clears the pool's entropy count and mixes some system
1771 * data into the pool to prepare it for use. The pool is not cleared
1772 * as that can only decrease the entropy in the pool.
1773 */
1774static void __init init_std_data(struct entropy_store *r)
1775{
1776	int i;
1777	ktime_t now = ktime_get_real();
1778	unsigned long rv;
1779
1780	mix_pool_bytes(r, &now, sizeof(now));
1781	for (i = r->poolinfo->poolbytes; i > 0; i -= sizeof(rv)) {
1782		if (!arch_get_random_seed_long(&rv) &&
1783		    !arch_get_random_long(&rv))
1784			rv = random_get_entropy();
1785		mix_pool_bytes(r, &rv, sizeof(rv));
1786	}
1787	mix_pool_bytes(r, utsname(), sizeof(*(utsname())));
1788}
1789
1790/*
1791 * Note that setup_arch() may call add_device_randomness()
1792 * long before we get here. This allows seeding of the pools
1793 * with some platform dependent data very early in the boot
1794 * process. But it limits our options here. We must use
1795 * statically allocated structures that already have all
1796 * initializations complete at compile time. We should also
1797 * take care not to overwrite the precious per platform data
1798 * we were given.
1799 */
1800int __init rand_initialize(void)
1801{
1802	init_std_data(&input_pool);
1803	crng_initialize_primary(&primary_crng);
1804	crng_global_init_time = jiffies;
1805	if (ratelimit_disable) {
1806		urandom_warning.interval = 0;
1807		unseeded_warning.interval = 0;
1808	}
1809	return 0;
1810}
1811
1812#ifdef CONFIG_BLOCK
1813void rand_initialize_disk(struct gendisk *disk)
1814{
1815	struct timer_rand_state *state;
1816
1817	/*
1818	 * If kzalloc returns null, we just won't use that entropy
1819	 * source.
1820	 */
1821	state = kzalloc(sizeof(struct timer_rand_state), GFP_KERNEL);
1822	if (state) {
1823		state->last_time = INITIAL_JIFFIES;
1824		disk->random = state;
1825	}
1826}
1827#endif
1828
1829static ssize_t
1830urandom_read_nowarn(struct file *file, char __user *buf, size_t nbytes,
1831		    loff_t *ppos)
1832{
1833	int ret;
1834
1835	nbytes = min_t(size_t, nbytes, INT_MAX >> (ENTROPY_SHIFT + 3));
1836	ret = extract_crng_user(buf, nbytes);
1837	trace_urandom_read(8 * nbytes, 0, ENTROPY_BITS(&input_pool));
1838	return ret;
1839}
1840
1841static ssize_t
1842urandom_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos)
1843{
1844	unsigned long flags;
1845	static int maxwarn = 10;
1846
1847	if (!crng_ready() && maxwarn > 0) {
1848		maxwarn--;
1849		if (__ratelimit(&urandom_warning))
1850			pr_notice("%s: uninitialized urandom read (%zd bytes read)\n",
1851				  current->comm, nbytes);
1852		spin_lock_irqsave(&primary_crng.lock, flags);
1853		crng_init_cnt = 0;
1854		spin_unlock_irqrestore(&primary_crng.lock, flags);
1855	}
1856
1857	return urandom_read_nowarn(file, buf, nbytes, ppos);
1858}
1859
1860static ssize_t
1861random_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos)
1862{
1863	int ret;
1864
1865	ret = wait_for_random_bytes();
1866	if (ret != 0)
1867		return ret;
1868	return urandom_read_nowarn(file, buf, nbytes, ppos);
1869}
1870
1871static __poll_t
1872random_poll(struct file *file, poll_table * wait)
1873{
1874	__poll_t mask;
1875
1876	poll_wait(file, &crng_init_wait, wait);
1877	poll_wait(file, &random_write_wait, wait);
1878	mask = 0;
1879	if (crng_ready())
1880		mask |= EPOLLIN | EPOLLRDNORM;
1881	if (ENTROPY_BITS(&input_pool) < random_write_wakeup_bits)
1882		mask |= EPOLLOUT | EPOLLWRNORM;
1883	return mask;
1884}
1885
1886static int
1887write_pool(struct entropy_store *r, const char __user *buffer, size_t count)
1888{
1889	size_t bytes;
1890	__u32 t, buf[16];
1891	const char __user *p = buffer;
1892
1893	while (count > 0) {
1894		int b, i = 0;
1895
1896		bytes = min(count, sizeof(buf));
1897		if (copy_from_user(&buf, p, bytes))
1898			return -EFAULT;
1899
1900		for (b = bytes ; b > 0 ; b -= sizeof(__u32), i++) {
1901			if (!arch_get_random_int(&t))
1902				break;
1903			buf[i] ^= t;
1904		}
1905
1906		count -= bytes;
1907		p += bytes;
1908
1909		mix_pool_bytes(r, buf, bytes);
1910		cond_resched();
1911	}
1912
1913	return 0;
1914}
1915
1916static ssize_t random_write(struct file *file, const char __user *buffer,
1917			    size_t count, loff_t *ppos)
1918{
1919	size_t ret;
1920
1921	ret = write_pool(&input_pool, buffer, count);
1922	if (ret)
1923		return ret;
1924
1925	return (ssize_t)count;
1926}
1927
1928static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
1929{
1930	int size, ent_count;
1931	int __user *p = (int __user *)arg;
1932	int retval;
1933
1934	switch (cmd) {
1935	case RNDGETENTCNT:
1936		/* inherently racy, no point locking */
1937		ent_count = ENTROPY_BITS(&input_pool);
1938		if (put_user(ent_count, p))
1939			return -EFAULT;
1940		return 0;
1941	case RNDADDTOENTCNT:
1942		if (!capable(CAP_SYS_ADMIN))
1943			return -EPERM;
1944		if (get_user(ent_count, p))
1945			return -EFAULT;
1946		return credit_entropy_bits_safe(&input_pool, ent_count);
1947	case RNDADDENTROPY:
1948		if (!capable(CAP_SYS_ADMIN))
1949			return -EPERM;
1950		if (get_user(ent_count, p++))
1951			return -EFAULT;
1952		if (ent_count < 0)
1953			return -EINVAL;
1954		if (get_user(size, p++))
1955			return -EFAULT;
1956		retval = write_pool(&input_pool, (const char __user *)p,
1957				    size);
1958		if (retval < 0)
1959			return retval;
1960		return credit_entropy_bits_safe(&input_pool, ent_count);
1961	case RNDZAPENTCNT:
1962	case RNDCLEARPOOL:
1963		/*
1964		 * Clear the entropy pool counters. We no longer clear
1965		 * the entropy pool, as that's silly.
1966		 */
1967		if (!capable(CAP_SYS_ADMIN))
1968			return -EPERM;
1969		input_pool.entropy_count = 0;
1970		return 0;
1971	case RNDRESEEDCRNG:
1972		if (!capable(CAP_SYS_ADMIN))
1973			return -EPERM;
1974		if (crng_init < 2)
1975			return -ENODATA;
1976		crng_reseed(&primary_crng, NULL);
1977		crng_global_init_time = jiffies - 1;
1978		return 0;
1979	default:
1980		return -EINVAL;
1981	}
1982}
1983
1984static int random_fasync(int fd, struct file *filp, int on)
1985{
1986	return fasync_helper(fd, filp, on, &fasync);
1987}
1988
1989const struct file_operations random_fops = {
1990	.read  = random_read,
1991	.write = random_write,
1992	.poll  = random_poll,
1993	.unlocked_ioctl = random_ioctl,
1994	.compat_ioctl = compat_ptr_ioctl,
1995	.fasync = random_fasync,
1996	.llseek = noop_llseek,
1997};
1998
1999const struct file_operations urandom_fops = {
2000	.read  = urandom_read,
2001	.write = random_write,
2002	.unlocked_ioctl = random_ioctl,
2003	.compat_ioctl = compat_ptr_ioctl,
2004	.fasync = random_fasync,
2005	.llseek = noop_llseek,
2006};
2007
2008SYSCALL_DEFINE3(getrandom, char __user *, buf, size_t, count,
2009		unsigned int, flags)
2010{
2011	int ret;
2012
2013	if (flags & ~(GRND_NONBLOCK|GRND_RANDOM|GRND_INSECURE))
2014		return -EINVAL;
2015
2016	/*
2017	 * Requesting insecure and blocking randomness at the same time makes
2018	 * no sense.
2019	 */
2020	if ((flags & (GRND_INSECURE|GRND_RANDOM)) == (GRND_INSECURE|GRND_RANDOM))
2021		return -EINVAL;
2022
2023	if (count > INT_MAX)
2024		count = INT_MAX;
2025
2026	if (!(flags & GRND_INSECURE) && !crng_ready()) {
2027		if (flags & GRND_NONBLOCK)
2028			return -EAGAIN;
2029		ret = wait_for_random_bytes();
2030		if (unlikely(ret))
2031			return ret;
2032	}
2033	return urandom_read_nowarn(NULL, buf, count, NULL);
2034}
2035
2036/********************************************************************
2037 *
2038 * Sysctl interface
2039 *
2040 ********************************************************************/
2041
2042#ifdef CONFIG_SYSCTL
2043
2044#include <linux/sysctl.h>
2045
2046static int min_write_thresh;
2047static int max_write_thresh = INPUT_POOL_WORDS * 32;
2048static int random_min_urandom_seed = 60;
2049static char sysctl_bootid[16];
2050
2051/*
2052 * This function is used to return both the bootid UUID, and random
2053 * UUID.  The difference is in whether table->data is NULL; if it is,
2054 * then a new UUID is generated and returned to the user.
2055 *
2056 * If the user accesses this via the proc interface, the UUID will be
2057 * returned as an ASCII string in the standard UUID format; if via the
2058 * sysctl system call, as 16 bytes of binary data.
2059 */
2060static int proc_do_uuid(struct ctl_table *table, int write,
2061			void *buffer, size_t *lenp, loff_t *ppos)
2062{
2063	struct ctl_table fake_table;
2064	unsigned char buf[64], tmp_uuid[16], *uuid;
2065
2066	uuid = table->data;
2067	if (!uuid) {
2068		uuid = tmp_uuid;
2069		generate_random_uuid(uuid);
2070	} else {
2071		static DEFINE_SPINLOCK(bootid_spinlock);
2072
2073		spin_lock(&bootid_spinlock);
2074		if (!uuid[8])
2075			generate_random_uuid(uuid);
2076		spin_unlock(&bootid_spinlock);
2077	}
2078
2079	sprintf(buf, "%pU", uuid);
2080
2081	fake_table.data = buf;
2082	fake_table.maxlen = sizeof(buf);
2083
2084	return proc_dostring(&fake_table, write, buffer, lenp, ppos);
2085}
2086
2087/*
2088 * Return entropy available scaled to integral bits
2089 */
2090static int proc_do_entropy(struct ctl_table *table, int write,
2091			   void *buffer, size_t *lenp, loff_t *ppos)
2092{
2093	struct ctl_table fake_table;
2094	int entropy_count;
2095
2096	entropy_count = *(int *)table->data >> ENTROPY_SHIFT;
2097
2098	fake_table.data = &entropy_count;
2099	fake_table.maxlen = sizeof(entropy_count);
2100
2101	return proc_dointvec(&fake_table, write, buffer, lenp, ppos);
2102}
2103
2104static int sysctl_poolsize = INPUT_POOL_WORDS * 32;
2105extern struct ctl_table random_table[];
2106struct ctl_table random_table[] = {
2107	{
2108		.procname	= "poolsize",
2109		.data		= &sysctl_poolsize,
2110		.maxlen		= sizeof(int),
2111		.mode		= 0444,
2112		.proc_handler	= proc_dointvec,
2113	},
2114	{
2115		.procname	= "entropy_avail",
2116		.maxlen		= sizeof(int),
2117		.mode		= 0444,
2118		.proc_handler	= proc_do_entropy,
2119		.data		= &input_pool.entropy_count,
2120	},
2121	{
2122		.procname	= "write_wakeup_threshold",
2123		.data		= &random_write_wakeup_bits,
2124		.maxlen		= sizeof(int),
2125		.mode		= 0644,
2126		.proc_handler	= proc_dointvec_minmax,
2127		.extra1		= &min_write_thresh,
2128		.extra2		= &max_write_thresh,
2129	},
2130	{
2131		.procname	= "urandom_min_reseed_secs",
2132		.data		= &random_min_urandom_seed,
2133		.maxlen		= sizeof(int),
2134		.mode		= 0644,
2135		.proc_handler	= proc_dointvec,
2136	},
2137	{
2138		.procname	= "boot_id",
2139		.data		= &sysctl_bootid,
2140		.maxlen		= 16,
2141		.mode		= 0444,
2142		.proc_handler	= proc_do_uuid,
2143	},
2144	{
2145		.procname	= "uuid",
2146		.maxlen		= 16,
2147		.mode		= 0444,
2148		.proc_handler	= proc_do_uuid,
2149	},
2150#ifdef ADD_INTERRUPT_BENCH
2151	{
2152		.procname	= "add_interrupt_avg_cycles",
2153		.data		= &avg_cycles,
2154		.maxlen		= sizeof(avg_cycles),
2155		.mode		= 0444,
2156		.proc_handler	= proc_doulongvec_minmax,
2157	},
2158	{
2159		.procname	= "add_interrupt_avg_deviation",
2160		.data		= &avg_deviation,
2161		.maxlen		= sizeof(avg_deviation),
2162		.mode		= 0444,
2163		.proc_handler	= proc_doulongvec_minmax,
2164	},
2165#endif
2166	{ }
2167};
2168#endif 	/* CONFIG_SYSCTL */
2169
2170struct batched_entropy {
2171	union {
2172		u64 entropy_u64[CHACHA_BLOCK_SIZE / sizeof(u64)];
2173		u32 entropy_u32[CHACHA_BLOCK_SIZE / sizeof(u32)];
2174	};
2175	unsigned int position;
2176	spinlock_t batch_lock;
2177};
2178
2179/*
2180 * Get a random word for internal kernel use only. The quality of the random
2181 * number is good as /dev/urandom, but there is no backtrack protection, with
2182 * the goal of being quite fast and not depleting entropy. In order to ensure
2183 * that the randomness provided by this function is okay, the function
2184 * wait_for_random_bytes() should be called and return 0 at least once at any
2185 * point prior.
2186 */
2187static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = {
2188	.batch_lock	= __SPIN_LOCK_UNLOCKED(batched_entropy_u64.lock),
2189};
2190
2191u64 get_random_u64(void)
2192{
2193	u64 ret;
2194	unsigned long flags;
2195	struct batched_entropy *batch;
2196	static void *previous;
2197
2198	warn_unseeded_randomness(&previous);
2199
2200	batch = raw_cpu_ptr(&batched_entropy_u64);
2201	spin_lock_irqsave(&batch->batch_lock, flags);
2202	if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) {
2203		extract_crng((u8 *)batch->entropy_u64);
2204		batch->position = 0;
2205	}
2206	ret = batch->entropy_u64[batch->position++];
2207	spin_unlock_irqrestore(&batch->batch_lock, flags);
2208	return ret;
2209}
2210EXPORT_SYMBOL(get_random_u64);
2211
2212static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32) = {
2213	.batch_lock	= __SPIN_LOCK_UNLOCKED(batched_entropy_u32.lock),
2214};
2215u32 get_random_u32(void)
2216{
2217	u32 ret;
2218	unsigned long flags;
2219	struct batched_entropy *batch;
2220	static void *previous;
2221
2222	warn_unseeded_randomness(&previous);
2223
2224	batch = raw_cpu_ptr(&batched_entropy_u32);
2225	spin_lock_irqsave(&batch->batch_lock, flags);
2226	if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) {
2227		extract_crng((u8 *)batch->entropy_u32);
2228		batch->position = 0;
2229	}
2230	ret = batch->entropy_u32[batch->position++];
2231	spin_unlock_irqrestore(&batch->batch_lock, flags);
2232	return ret;
2233}
2234EXPORT_SYMBOL(get_random_u32);
2235
2236/* It's important to invalidate all potential batched entropy that might
2237 * be stored before the crng is initialized, which we can do lazily by
2238 * simply resetting the counter to zero so that it's re-extracted on the
2239 * next usage. */
2240static void invalidate_batched_entropy(void)
2241{
2242	int cpu;
2243	unsigned long flags;
2244
2245	for_each_possible_cpu (cpu) {
2246		struct batched_entropy *batched_entropy;
2247
2248		batched_entropy = per_cpu_ptr(&batched_entropy_u32, cpu);
2249		spin_lock_irqsave(&batched_entropy->batch_lock, flags);
2250		batched_entropy->position = 0;
2251		spin_unlock(&batched_entropy->batch_lock);
2252
2253		batched_entropy = per_cpu_ptr(&batched_entropy_u64, cpu);
2254		spin_lock(&batched_entropy->batch_lock);
2255		batched_entropy->position = 0;
2256		spin_unlock_irqrestore(&batched_entropy->batch_lock, flags);
2257	}
2258}
2259
2260/**
2261 * randomize_page - Generate a random, page aligned address
2262 * @start:	The smallest acceptable address the caller will take.
2263 * @range:	The size of the area, starting at @start, within which the
2264 *		random address must fall.
2265 *
2266 * If @start + @range would overflow, @range is capped.
2267 *
2268 * NOTE: Historical use of randomize_range, which this replaces, presumed that
2269 * @start was already page aligned.  We now align it regardless.
2270 *
2271 * Return: A page aligned address within [start, start + range).  On error,
2272 * @start is returned.
2273 */
2274unsigned long
2275randomize_page(unsigned long start, unsigned long range)
2276{
2277	if (!PAGE_ALIGNED(start)) {
2278		range -= PAGE_ALIGN(start) - start;
2279		start = PAGE_ALIGN(start);
2280	}
2281
2282	if (start > ULONG_MAX - range)
2283		range = ULONG_MAX - start;
2284
2285	range >>= PAGE_SHIFT;
2286
2287	if (range == 0)
2288		return start;
2289
2290	return start + (get_random_long() % range << PAGE_SHIFT);
2291}
2292
2293/* Interface for in-kernel drivers of true hardware RNGs.
2294 * Those devices may produce endless random bits and will be throttled
2295 * when our pool is full.
2296 */
2297void add_hwgenerator_randomness(const char *buffer, size_t count,
2298				size_t entropy)
2299{
2300	struct entropy_store *poolp = &input_pool;
2301
2302	if (unlikely(crng_init == 0)) {
2303		crng_fast_load(buffer, count);
2304		return;
2305	}
2306
2307	/* Suspend writing if we're above the trickle threshold.
2308	 * We'll be woken up again once below random_write_wakeup_thresh,
2309	 * or when the calling thread is about to terminate.
2310	 */
2311	wait_event_interruptible(random_write_wait, kthread_should_stop() ||
2312			ENTROPY_BITS(&input_pool) <= random_write_wakeup_bits);
2313	mix_pool_bytes(poolp, buffer, count);
2314	credit_entropy_bits(poolp, entropy);
2315}
2316EXPORT_SYMBOL_GPL(add_hwgenerator_randomness);
2317
2318/* Handle random seed passed by bootloader.
2319 * If the seed is trustworthy, it would be regarded as hardware RNGs. Otherwise
2320 * it would be regarded as device data.
2321 * The decision is controlled by CONFIG_RANDOM_TRUST_BOOTLOADER.
2322 */
2323void add_bootloader_randomness(const void *buf, unsigned int size)
2324{
2325	if (IS_ENABLED(CONFIG_RANDOM_TRUST_BOOTLOADER))
2326		add_hwgenerator_randomness(buf, size, size * 8);
2327	else
2328		add_device_randomness(buf, size);
2329}
2330EXPORT_SYMBOL_GPL(add_bootloader_randomness);