Merge tag 'random-6.1-rc1-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/crng/random

+2 -2

drivers/char/mem.c

··· 712 712 #endif 713 713 [5] = { "zero", 0666, &zero_fops, FMODE_NOWAIT }, 714 714 [7] = { "full", 0666, &full_fops, 0 }, 715 - [8] = { "random", 0666, &random_fops, 0 }, 716 - [9] = { "urandom", 0666, &urandom_fops, 0 }, 715 + [8] = { "random", 0666, &random_fops, FMODE_NOWAIT }, 716 + [9] = { "urandom", 0666, &urandom_fops, FMODE_NOWAIT }, 717 717 #ifdef CONFIG_PRINTK 718 718 [11] = { "kmsg", 0644, &kmsg_fops, 0 }, 719 719 #endif

+81 -58

drivers/char/random.c

··· 96 96 /* 97 97 * Returns whether or not the input pool has been seeded and thus guaranteed 98 98 * to supply cryptographically secure random numbers. This applies to: the 99 - * /dev/urandom device, the get_random_bytes function, and the get_random_{u32, 100 - * ,u64,int,long} family of functions. 99 + * /dev/urandom device, the get_random_bytes function, and the get_random_{u8, 100 + * u16,u32,u64,int,long} family of functions. 101 101 * 102 102 * Returns: true if the input pool has been seeded. 103 103 * false if the input pool has not been seeded. ··· 119 119 /* 120 120 * Wait for the input pool to be seeded and thus guaranteed to supply 121 121 * cryptographically secure random numbers. This applies to: the /dev/urandom 122 - * device, the get_random_bytes function, and the get_random_{u32,u64,int,long} 123 - * family of functions. Using any of these functions without first calling 124 - * this function forfeits the guarantee of security. 122 + * device, the get_random_bytes function, and the get_random_{u8,u16,u32,u64, 123 + * int,long} family of functions. Using any of these functions without first 124 + * calling this function forfeits the guarantee of security. 125 125 * 126 126 * Returns: 0 if the input pool has been seeded. 127 127 * -ERESTARTSYS if the function was interrupted by a signal. ··· 157 157 * There are a few exported interfaces for use by other drivers: 158 158 * 159 159 * void get_random_bytes(void *buf, size_t len) 160 + * u8 get_random_u8() 161 + * u16 get_random_u16() 160 162 * u32 get_random_u32() 161 163 * u64 get_random_u64() 162 164 * unsigned int get_random_int() ··· 166 164 * 167 165 * These interfaces will return the requested number of random bytes 168 166 * into the given buffer or as a return value. This is equivalent to 169 - * a read from /dev/urandom. The u32, u64, int, and long family of 170 - * functions may be higher performance for one-off random integers, 171 - * because they do a bit of buffering and do not invoke reseeding 172 - * until the buffer is emptied. 167 + * a read from /dev/urandom. The u8, u16, u32, u64, int, and long 168 + * family of functions may be higher performance for one-off random 169 + * integers, because they do a bit of buffering and do not invoke 170 + * reseeding until the buffer is emptied. 173 171 * 174 172 *********************************************************************/ 175 173 ··· 262 260 } 263 261 264 262 /* 265 - * Return whether the crng seed is considered to be sufficiently old 266 - * that a reseeding is needed. This happens if the last reseeding 267 - * was CRNG_RESEED_INTERVAL ago, or during early boot, at an interval 263 + * Return the interval until the next reseeding, which is normally 264 + * CRNG_RESEED_INTERVAL, but during early boot, it is at an interval 268 265 * proportional to the uptime. 269 266 */ 270 - static bool crng_has_old_seed(void) 267 + static unsigned int crng_reseed_interval(void) 271 268 { 272 269 static bool early_boot = true; 273 - unsigned long interval = CRNG_RESEED_INTERVAL; 274 270 275 271 if (unlikely(READ_ONCE(early_boot))) { 276 272 time64_t uptime = ktime_get_seconds(); 277 273 if (uptime >= CRNG_RESEED_INTERVAL / HZ * 2) 278 274 WRITE_ONCE(early_boot, false); 279 275 else 280 - interval = max_t(unsigned int, CRNG_RESEED_START_INTERVAL, 281 - (unsigned int)uptime / 2 * HZ); 276 + return max_t(unsigned int, CRNG_RESEED_START_INTERVAL, 277 + (unsigned int)uptime / 2 * HZ); 282 278 } 283 - return time_is_before_jiffies(READ_ONCE(base_crng.birth) + interval); 279 + return CRNG_RESEED_INTERVAL; 284 280 } 285 281 286 282 /* ··· 320 320 * If the base_crng is old enough, we reseed, which in turn bumps the 321 321 * generation counter that we check below. 322 322 */ 323 - if (unlikely(crng_has_old_seed())) 323 + if (unlikely(time_is_before_jiffies(READ_ONCE(base_crng.birth) + crng_reseed_interval()))) 324 324 crng_reseed(); 325 325 326 326 local_lock_irqsave(&crngs.lock, flags); ··· 384 384 } 385 385 386 386 /* 387 - * This function is the exported kernel interface. It returns some 388 - * number of good random numbers, suitable for key generation, seeding 389 - * TCP sequence numbers, etc. In order to ensure that the randomness 390 - * by this function is okay, the function wait_for_random_bytes() 391 - * should be called and return 0 at least once at any point prior. 387 + * This function is the exported kernel interface. It returns some number of 388 + * good random numbers, suitable for key generation, seeding TCP sequence 389 + * numbers, etc. In order to ensure that the randomness returned by this 390 + * function is okay, the function wait_for_random_bytes() should be called and 391 + * return 0 at least once at any point prior. 392 392 */ 393 393 void get_random_bytes(void *buf, size_t len) 394 394 { ··· 506 506 } \ 507 507 EXPORT_SYMBOL(get_random_ ##type); 508 508 509 - DEFINE_BATCHED_ENTROPY(u64) 509 + DEFINE_BATCHED_ENTROPY(u8) 510 + DEFINE_BATCHED_ENTROPY(u16) 510 511 DEFINE_BATCHED_ENTROPY(u32) 512 + DEFINE_BATCHED_ENTROPY(u64) 511 513 512 514 #ifdef CONFIG_SMP 513 515 /* ··· 524 522 * randomness. 525 523 */ 526 524 per_cpu_ptr(&crngs, cpu)->generation = ULONG_MAX; 525 + per_cpu_ptr(&batched_entropy_u8, cpu)->position = UINT_MAX; 526 + per_cpu_ptr(&batched_entropy_u16, cpu)->position = UINT_MAX; 527 527 per_cpu_ptr(&batched_entropy_u32, cpu)->position = UINT_MAX; 528 528 per_cpu_ptr(&batched_entropy_u64, cpu)->position = UINT_MAX; 529 529 return 0; ··· 778 774 static struct notifier_block pm_notifier = { .notifier_call = random_pm_notification }; 779 775 780 776 /* 781 - * The first collection of entropy occurs at system boot while interrupts 782 - * are still turned off. Here we push in latent entropy, RDSEED, a timestamp, 783 - * utsname(), and the command line. Depending on the above configuration knob, 784 - * RDSEED may be considered sufficient for initialization. Note that much 785 - * earlier setup may already have pushed entropy into the input pool by the 786 - * time we get here. 777 + * This is called extremely early, before time keeping functionality is 778 + * available, but arch randomness is. Interrupts are not yet enabled. 787 779 */ 788 - int __init random_init(const char *command_line) 780 + void __init random_init_early(const char *command_line) 789 781 { 790 - ktime_t now = ktime_get_real(); 791 - size_t i, longs, arch_bits; 792 782 unsigned long entropy[BLAKE2S_BLOCK_SIZE / sizeof(long)]; 783 + size_t i, longs, arch_bits; 793 784 794 785 #if defined(LATENT_ENTROPY_PLUGIN) 795 786 static const u8 compiletime_seed[BLAKE2S_BLOCK_SIZE] __initconst __latent_entropy; ··· 804 805 i += longs; 805 806 continue; 806 807 } 807 - entropy[0] = random_get_entropy(); 808 - _mix_pool_bytes(entropy, sizeof(*entropy)); 809 808 arch_bits -= sizeof(*entropy) * 8; 810 809 ++i; 811 810 } 812 - _mix_pool_bytes(&now, sizeof(now)); 813 - _mix_pool_bytes(utsname(), sizeof(*(utsname()))); 811 + 812 + _mix_pool_bytes(init_utsname(), sizeof(*(init_utsname()))); 814 813 _mix_pool_bytes(command_line, strlen(command_line)); 814 + 815 + /* Reseed if already seeded by earlier phases. */ 816 + if (crng_ready()) 817 + crng_reseed(); 818 + else if (trust_cpu) 819 + _credit_init_bits(arch_bits); 820 + } 821 + 822 + /* 823 + * This is called a little bit after the prior function, and now there is 824 + * access to timestamps counters. Interrupts are not yet enabled. 825 + */ 826 + void __init random_init(void) 827 + { 828 + unsigned long entropy = random_get_entropy(); 829 + ktime_t now = ktime_get_real(); 830 + 831 + _mix_pool_bytes(&now, sizeof(now)); 832 + _mix_pool_bytes(&entropy, sizeof(entropy)); 815 833 add_latent_entropy(); 816 834 817 835 /* 818 - * If we were initialized by the bootloader before jump labels are 819 - * initialized, then we should enable the static branch here, where 836 + * If we were initialized by the cpu or bootloader before jump labels 837 + * are initialized, then we should enable the static branch here, where 820 838 * it's guaranteed that jump labels have been initialized. 821 839 */ 822 840 if (!static_branch_likely(&crng_is_ready) && crng_init >= CRNG_READY) 823 841 crng_set_ready(NULL); 824 842 843 + /* Reseed if already seeded by earlier phases. */ 825 844 if (crng_ready()) 826 845 crng_reseed(); 827 - else if (trust_cpu) 828 - _credit_init_bits(arch_bits); 829 846 830 847 WARN_ON(register_pm_notifier(&pm_notifier)); 831 848 832 - WARN(!random_get_entropy(), "Missing cycle counter and fallback timer; RNG " 833 - "entropy collection will consequently suffer."); 834 - return 0; 849 + WARN(!entropy, "Missing cycle counter and fallback timer; RNG " 850 + "entropy collection will consequently suffer."); 835 851 } 836 852 837 853 /* ··· 880 866 credit_init_bits(entropy); 881 867 882 868 /* 883 - * Throttle writing to once every CRNG_RESEED_INTERVAL, unless 884 - * we're not yet initialized. 869 + * Throttle writing to once every reseed interval, unless we're not yet 870 + * initialized or no entropy is credited. 885 871 */ 886 - if (!kthread_should_stop() && crng_ready()) 887 - schedule_timeout_interruptible(CRNG_RESEED_INTERVAL); 872 + if (!kthread_should_stop() && (crng_ready() || !entropy)) 873 + schedule_timeout_interruptible(crng_reseed_interval()); 888 874 } 889 875 EXPORT_SYMBOL_GPL(add_hwgenerator_randomness); 890 876 ··· 934 920 #endif 935 921 936 922 struct fast_pool { 937 - struct work_struct mix; 938 923 unsigned long pool[4]; 939 924 unsigned long last; 940 925 unsigned int count; 926 + struct timer_list mix; 941 927 }; 928 + 929 + static void mix_interrupt_randomness(struct timer_list *work); 942 930 943 931 static DEFINE_PER_CPU(struct fast_pool, irq_randomness) = { 944 932 #ifdef CONFIG_64BIT 945 933 #define FASTMIX_PERM SIPHASH_PERMUTATION 946 - .pool = { SIPHASH_CONST_0, SIPHASH_CONST_1, SIPHASH_CONST_2, SIPHASH_CONST_3 } 934 + .pool = { SIPHASH_CONST_0, SIPHASH_CONST_1, SIPHASH_CONST_2, SIPHASH_CONST_3 }, 947 935 #else 948 936 #define FASTMIX_PERM HSIPHASH_PERMUTATION 949 - .pool = { HSIPHASH_CONST_0, HSIPHASH_CONST_1, HSIPHASH_CONST_2, HSIPHASH_CONST_3 } 937 + .pool = { HSIPHASH_CONST_0, HSIPHASH_CONST_1, HSIPHASH_CONST_2, HSIPHASH_CONST_3 }, 950 938 #endif 939 + .mix = __TIMER_INITIALIZER(mix_interrupt_randomness, 0) 951 940 }; 952 941 953 942 /* ··· 992 975 } 993 976 #endif 994 977 995 - static void mix_interrupt_randomness(struct work_struct *work) 978 + static void mix_interrupt_randomness(struct timer_list *work) 996 979 { 997 980 struct fast_pool *fast_pool = container_of(work, struct fast_pool, mix); 998 981 /* ··· 1023 1006 local_irq_enable(); 1024 1007 1025 1008 mix_pool_bytes(pool, sizeof(pool)); 1026 - credit_init_bits(max(1u, (count & U16_MAX) / 64)); 1009 + credit_init_bits(clamp_t(unsigned int, (count & U16_MAX) / 64, 1, sizeof(pool) * 8)); 1027 1010 1028 1011 memzero_explicit(pool, sizeof(pool)); 1029 1012 } ··· 1046 1029 if (new_count < 1024 && !time_is_before_jiffies(fast_pool->last + HZ)) 1047 1030 return; 1048 1031 1049 - if (unlikely(!fast_pool->mix.func)) 1050 - INIT_WORK(&fast_pool->mix, mix_interrupt_randomness); 1051 1032 fast_pool->count |= MIX_INFLIGHT; 1052 - queue_work_on(raw_smp_processor_id(), system_highpri_wq, &fast_pool->mix); 1033 + if (!timer_pending(&fast_pool->mix)) { 1034 + fast_pool->mix.expires = jiffies; 1035 + add_timer_on(&fast_pool->mix, raw_smp_processor_id()); 1036 + } 1053 1037 } 1054 1038 EXPORT_SYMBOL_GPL(add_interrupt_randomness); 1055 1039 ··· 1209 1191 */ 1210 1192 static void __cold try_to_generate_entropy(void) 1211 1193 { 1212 - enum { NUM_TRIAL_SAMPLES = 8192, MAX_SAMPLES_PER_BIT = HZ / 30 }; 1194 + enum { NUM_TRIAL_SAMPLES = 8192, MAX_SAMPLES_PER_BIT = HZ / 15 }; 1213 1195 struct entropy_timer_state stack; 1214 1196 unsigned int i, num_different = 0; 1215 1197 unsigned long last = random_get_entropy(); ··· 1228 1210 timer_setup_on_stack(&stack.timer, entropy_timer, 0); 1229 1211 while (!crng_ready() && !signal_pending(current)) { 1230 1212 if (!timer_pending(&stack.timer)) 1231 - mod_timer(&stack.timer, jiffies + 1); 1213 + mod_timer(&stack.timer, jiffies); 1232 1214 mix_pool_bytes(&stack.entropy, sizeof(stack.entropy)); 1233 1215 schedule(); 1234 1216 stack.entropy = random_get_entropy(); ··· 1364 1346 static ssize_t random_read_iter(struct kiocb *kiocb, struct iov_iter *iter) 1365 1347 { 1366 1348 int ret; 1349 + 1350 + if (!crng_ready() && 1351 + ((kiocb->ki_flags & (IOCB_NOWAIT | IOCB_NOIO)) || 1352 + (kiocb->ki_filp->f_flags & O_NONBLOCK))) 1353 + return -EAGAIN; 1367 1354 1368 1355 ret = wait_for_random_bytes(); 1369 1356 if (ret != 0)

+11 -6

include/linux/prandom.h

··· 12 12 #include <linux/percpu.h> 13 13 #include <linux/random.h> 14 14 15 + /* Deprecated: use get_random_u32 instead. */ 15 16 static inline u32 prandom_u32(void) 16 17 { 17 18 return get_random_u32(); 18 19 } 19 20 21 + /* Deprecated: use get_random_bytes instead. */ 20 22 static inline void prandom_bytes(void *buf, size_t nbytes) 21 23 { 22 24 return get_random_bytes(buf, nbytes); ··· 39 37 * prandom_u32_max - returns a pseudo-random number in interval [0, ep_ro) 40 38 * @ep_ro: right open interval endpoint 41 39 * 42 - * Returns a pseudo-random number that is in interval [0, ep_ro). Note 43 - * that the result depends on PRNG being well distributed in [0, ~0U] 44 - * u32 space. Here we use maximally equidistributed combined Tausworthe 45 - * generator, that is, prandom_u32(). This is useful when requesting a 46 - * random index of an array containing ep_ro elements, for example. 40 + * Returns a pseudo-random number that is in interval [0, ep_ro). This is 41 + * useful when requesting a random index of an array containing ep_ro elements, 42 + * for example. The result is somewhat biased when ep_ro is not a power of 2, 43 + * so do not use this for cryptographic purposes. 47 44 * 48 45 * Returns: pseudo-random number in interval [0, ep_ro) 49 46 */ 50 47 static inline u32 prandom_u32_max(u32 ep_ro) 51 48 { 52 - return (u32)(((u64) prandom_u32() * ep_ro) >> 32); 49 + if (__builtin_constant_p(ep_ro <= 1U << 8) && ep_ro <= 1U << 8) 50 + return (get_random_u8() * ep_ro) >> 8; 51 + if (__builtin_constant_p(ep_ro <= 1U << 16) && ep_ro <= 1U << 16) 52 + return (get_random_u16() * ep_ro) >> 16; 53 + return ((u64)get_random_u32() * ep_ro) >> 32; 53 54 } 54 55 55 56 /*

+6 -1

include/linux/random.h

··· 38 38 #endif 39 39 40 40 void get_random_bytes(void *buf, size_t len); 41 + u8 get_random_u8(void); 42 + u16 get_random_u16(void); 41 43 u32 get_random_u32(void); 42 44 u64 get_random_u64(void); 43 45 static inline unsigned int get_random_int(void) ··· 74 72 return get_random_long() & CANARY_MASK; 75 73 } 76 74 77 - int __init random_init(const char *command_line); 75 + void __init random_init_early(const char *command_line); 76 + void __init random_init(void); 78 77 bool rng_is_initialized(void); 79 78 int wait_for_random_bytes(void); 80 79 ··· 96 93 *out = get_random_ ## name(); \ 97 94 return 0; \ 98 95 } 96 + declare_get_random_var_wait(u8, u8) 97 + declare_get_random_var_wait(u16, u16) 99 98 declare_get_random_var_wait(u32, u32) 100 99 declare_get_random_var_wait(u64, u32) 101 100 declare_get_random_var_wait(int, unsigned int)

+8 -9

init/main.c

··· 976 976 parse_args("Setting extra init args", extra_init_args, 977 977 NULL, 0, -1, -1, NULL, set_init_arg); 978 978 979 + /* Architectural and non-timekeeping rng init, before allocator init */ 980 + random_init_early(command_line); 981 + 979 982 /* 980 983 * These use large bootmem allocations and must precede 981 984 * kmem_cache_init() ··· 1038 1035 hrtimers_init(); 1039 1036 softirq_init(); 1040 1037 timekeeping_init(); 1041 - kfence_init(); 1042 1038 time_init(); 1043 1039 1044 - /* 1045 - * For best initial stack canary entropy, prepare it after: 1046 - * - setup_arch() for any UEFI RNG entropy and boot cmdline access 1047 - * - timekeeping_init() for ktime entropy used in random_init() 1048 - * - time_init() for making random_get_entropy() work on some platforms 1049 - * - random_init() to initialize the RNG from from early entropy sources 1050 - */ 1051 - random_init(command_line); 1040 + /* This must be after timekeeping is initialized */ 1041 + random_init(); 1042 + 1043 + /* These make use of the fully initialized rng */ 1044 + kfence_init(); 1052 1045 boot_init_stack_canary(); 1053 1046 1054 1047 perf_event_init();

+3

kernel/sys.c

··· 25 25 #include <linux/times.h> 26 26 #include <linux/posix-timers.h> 27 27 #include <linux/security.h> 28 + #include <linux/random.h> 28 29 #include <linux/suspend.h> 29 30 #include <linux/tty.h> 30 31 #include <linux/signal.h> ··· 1367 1366 if (!copy_from_user(tmp, name, len)) { 1368 1367 struct new_utsname *u; 1369 1368 1369 + add_device_randomness(tmp, len); 1370 1370 down_write(&uts_sem); 1371 1371 u = utsname(); 1372 1372 memcpy(u->nodename, tmp, len); ··· 1421 1419 if (!copy_from_user(tmp, name, len)) { 1422 1420 struct new_utsname *u; 1423 1421 1422 + add_device_randomness(tmp, len); 1424 1423 down_write(&uts_sem); 1425 1424 u = utsname(); 1426 1425 memcpy(u->domainname, tmp, len);

+2

kernel/utsname_sysctl.c

··· 8 8 #include <linux/export.h> 9 9 #include <linux/uts.h> 10 10 #include <linux/utsname.h> 11 + #include <linux/random.h> 11 12 #include <linux/sysctl.h> 12 13 #include <linux/wait.h> 13 14 #include <linux/rwsem.h> ··· 58 57 * theoretically be incorrect if there are two parallel writes 59 58 * at non-zero offsets to the same sysctl. 60 59 */ 60 + add_device_randomness(tmp_data, sizeof(tmp_data)); 61 61 down_write(&uts_sem); 62 62 memcpy(get_uts(table), tmp_data, sizeof(tmp_data)); 63 63 up_write(&uts_sem);

+1 -1

mm/kfence/core.c

··· 864 864 865 865 void __init kfence_init(void) 866 866 { 867 - stack_hash_seed = (u32)random_get_entropy(); 867 + stack_hash_seed = get_random_u32(); 868 868 869 869 /* Setting kfence_sample_interval to 0 on boot disables KFENCE. */ 870 870 if (!kfence_sample_interval)

Configure Feed

Configure Feed