Merge tag 'fscrypt-for-linus' of git://git.kernel.org/pub/scm/fs/fscrypt/fscrypt

+10 -6

fs/crypto/bio.c

··· 25 25 * then this function isn't applicable. This function may sleep, so it must be 26 26 * called from a workqueue rather than from the bio's bi_end_io callback. 27 27 * 28 - * This function sets PG_error on any pages that contain any blocks that failed 29 - * to be decrypted. The filesystem must not mark such pages uptodate. 28 + * Return: %true on success; %false on failure. On failure, bio->bi_status is 29 + * also set to an error status. 30 30 */ 31 - void fscrypt_decrypt_bio(struct bio *bio) 31 + bool fscrypt_decrypt_bio(struct bio *bio) 32 32 { 33 33 struct bio_vec *bv; 34 34 struct bvec_iter_all iter_all; 35 35 36 36 bio_for_each_segment_all(bv, bio, iter_all) { 37 37 struct page *page = bv->bv_page; 38 - int ret = fscrypt_decrypt_pagecache_blocks(page, bv->bv_len, 38 + int err = fscrypt_decrypt_pagecache_blocks(page, bv->bv_len, 39 39 bv->bv_offset); 40 - if (ret) 41 - SetPageError(page); 40 + 41 + if (err) { 42 + bio->bi_status = errno_to_blk_status(err); 43 + return false; 44 + } 42 45 } 46 + return true; 43 47 } 44 48 EXPORT_SYMBOL(fscrypt_decrypt_bio); 45 49

+58 -24

fs/crypto/fscrypt_private.h

··· 184 184 struct fscrypt_prepared_key { 185 185 struct crypto_skcipher *tfm; 186 186 #ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT 187 - struct fscrypt_blk_crypto_key *blk_key; 187 + struct blk_crypto_key *blk_key; 188 188 #endif 189 189 }; 190 190 ··· 225 225 * will be NULL if the master key was found in a process-subscribed 226 226 * keyring rather than in the filesystem-level keyring. 227 227 */ 228 - struct key *ci_master_key; 228 + struct fscrypt_master_key *ci_master_key; 229 229 230 230 /* 231 231 * Link in list of inodes that were unlocked with the master key. ··· 344 344 const u8 *raw_key, 345 345 const struct fscrypt_info *ci); 346 346 347 - void fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key); 347 + void fscrypt_destroy_inline_crypt_key(struct super_block *sb, 348 + struct fscrypt_prepared_key *prep_key); 348 349 349 350 /* 350 351 * Check whether the crypto transform or blk-crypto key has been allocated in ··· 391 390 } 392 391 393 392 static inline void 394 - fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key) 393 + fscrypt_destroy_inline_crypt_key(struct super_block *sb, 394 + struct fscrypt_prepared_key *prep_key) 395 395 { 396 396 } 397 397 ··· 439 437 struct fscrypt_master_key { 440 438 441 439 /* 440 + * Back-pointer to the super_block of the filesystem to which this 441 + * master key has been added. Only valid if ->mk_active_refs > 0. 442 + */ 443 + struct super_block *mk_sb; 444 + 445 + /* 446 + * Link in ->mk_sb->s_master_keys->key_hashtable. 447 + * Only valid if ->mk_active_refs > 0. 448 + */ 449 + struct hlist_node mk_node; 450 + 451 + /* Semaphore that protects ->mk_secret and ->mk_users */ 452 + struct rw_semaphore mk_sem; 453 + 454 + /* 455 + * Active and structural reference counts. An active ref guarantees 456 + * that the struct continues to exist, continues to be in the keyring 457 + * ->mk_sb->s_master_keys, and that any embedded subkeys (e.g. 458 + * ->mk_direct_keys) that have been prepared continue to exist. 459 + * A structural ref only guarantees that the struct continues to exist. 460 + * 461 + * There is one active ref associated with ->mk_secret being present, 462 + * and one active ref for each inode in ->mk_decrypted_inodes. 463 + * 464 + * There is one structural ref associated with the active refcount being 465 + * nonzero. Finding a key in the keyring also takes a structural ref, 466 + * which is then held temporarily while the key is operated on. 467 + */ 468 + refcount_t mk_active_refs; 469 + refcount_t mk_struct_refs; 470 + 471 + struct rcu_head mk_rcu_head; 472 + 473 + /* 442 474 * The secret key material. After FS_IOC_REMOVE_ENCRYPTION_KEY is 443 475 * executed, this is wiped and no new inodes can be unlocked with this 444 476 * key; however, there may still be inodes in ->mk_decrypted_inodes ··· 480 444 * FS_IOC_REMOVE_ENCRYPTION_KEY can be retried, or 481 445 * FS_IOC_ADD_ENCRYPTION_KEY can add the secret again. 482 446 * 483 - * Locking: protected by this master key's key->sem. 447 + * While ->mk_secret is present, one ref in ->mk_active_refs is held. 448 + * 449 + * Locking: protected by ->mk_sem. The manipulation of ->mk_active_refs 450 + * associated with this field is protected by ->mk_sem as well. 484 451 */ 485 452 struct fscrypt_master_key_secret mk_secret; 486 453 ··· 504 465 * 505 466 * This is NULL for v1 policy keys; those can only be added by root. 506 467 * 507 - * Locking: in addition to this keyring's own semaphore, this is 508 - * protected by this master key's key->sem, so we can do atomic 509 - * search+insert. It can also be searched without taking any locks, but 510 - * in that case the returned key may have already been removed. 468 + * Locking: protected by ->mk_sem. (We don't just rely on the keyrings 469 + * subsystem semaphore ->mk_users->sem, as we need support for atomic 470 + * search+insert along with proper synchronization with ->mk_secret.) 511 471 */ 512 472 struct key *mk_users; 513 - 514 - /* 515 - * Length of ->mk_decrypted_inodes, plus one if mk_secret is present. 516 - * Once this goes to 0, the master key is removed from ->s_master_keys. 517 - * The 'struct fscrypt_master_key' will continue to live as long as the 518 - * 'struct key' whose payload it is, but we won't let this reference 519 - * count rise again. 520 - */ 521 - refcount_t mk_refcount; 522 473 523 474 /* 524 475 * List of inodes that were unlocked using this key. This allows the ··· 535 506 is_master_key_secret_present(const struct fscrypt_master_key_secret *secret) 536 507 { 537 508 /* 538 - * The READ_ONCE() is only necessary for fscrypt_drop_inode() and 539 - * fscrypt_key_describe(). These run in atomic context, so they can't 540 - * take the key semaphore and thus 'secret' can change concurrently 541 - * which would be a data race. But they only need to know whether the 509 + * The READ_ONCE() is only necessary for fscrypt_drop_inode(). 510 + * fscrypt_drop_inode() runs in atomic context, so it can't take the key 511 + * semaphore and thus 'secret' can change concurrently which would be a 512 + * data race. But fscrypt_drop_inode() only need to know whether the 542 513 * secret *was* present at the time of check, so READ_ONCE() suffices. 543 514 */ 544 515 return READ_ONCE(secret->size) != 0; ··· 567 538 return 0; 568 539 } 569 540 570 - struct key * 541 + void fscrypt_put_master_key(struct fscrypt_master_key *mk); 542 + 543 + void fscrypt_put_master_key_activeref(struct fscrypt_master_key *mk); 544 + 545 + struct fscrypt_master_key * 571 546 fscrypt_find_master_key(struct super_block *sb, 572 547 const struct fscrypt_key_specifier *mk_spec); 573 548 ··· 602 569 int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key, 603 570 const u8 *raw_key, const struct fscrypt_info *ci); 604 571 605 - void fscrypt_destroy_prepared_key(struct fscrypt_prepared_key *prep_key); 572 + void fscrypt_destroy_prepared_key(struct super_block *sb, 573 + struct fscrypt_prepared_key *prep_key); 606 574 607 575 int fscrypt_set_per_file_enc_key(struct fscrypt_info *ci, const u8 *raw_key); 608 576

+3 -7

fs/crypto/hooks.c

··· 5 5 * Encryption hooks for higher-level filesystem operations. 6 6 */ 7 7 8 - #include <linux/key.h> 9 - 10 8 #include "fscrypt_private.h" 11 9 12 10 /** ··· 140 142 unsigned int oldflags, unsigned int flags) 141 143 { 142 144 struct fscrypt_info *ci; 143 - struct key *key; 144 145 struct fscrypt_master_key *mk; 145 146 int err; 146 147 ··· 155 158 ci = inode->i_crypt_info; 156 159 if (ci->ci_policy.version != FSCRYPT_POLICY_V2) 157 160 return -EINVAL; 158 - key = ci->ci_master_key; 159 - mk = key->payload.data[0]; 160 - down_read(&key->sem); 161 + mk = ci->ci_master_key; 162 + down_read(&mk->mk_sem); 161 163 if (is_master_key_secret_present(&mk->mk_secret)) 162 164 err = fscrypt_derive_dirhash_key(ci, mk); 163 165 else 164 166 err = -ENOKEY; 165 - up_read(&key->sem); 167 + up_read(&mk->mk_sem); 166 168 return err; 167 169 } 168 170 return 0;

+72 -77

fs/crypto/inline_crypt.c

··· 21 21 22 22 #include "fscrypt_private.h" 23 23 24 - struct fscrypt_blk_crypto_key { 25 - struct blk_crypto_key base; 26 - int num_devs; 27 - struct request_queue *devs[]; 28 - }; 29 - 30 - static int fscrypt_get_num_devices(struct super_block *sb) 24 + static struct block_device **fscrypt_get_devices(struct super_block *sb, 25 + unsigned int *num_devs) 31 26 { 32 - if (sb->s_cop->get_num_devices) 33 - return sb->s_cop->get_num_devices(sb); 34 - return 1; 35 - } 27 + struct block_device **devs; 36 28 37 - static void fscrypt_get_devices(struct super_block *sb, int num_devs, 38 - struct request_queue **devs) 39 - { 40 - if (num_devs == 1) 41 - devs[0] = bdev_get_queue(sb->s_bdev); 42 - else 43 - sb->s_cop->get_devices(sb, devs); 29 + if (sb->s_cop->get_devices) { 30 + devs = sb->s_cop->get_devices(sb, num_devs); 31 + if (devs) 32 + return devs; 33 + } 34 + devs = kmalloc(sizeof(*devs), GFP_KERNEL); 35 + if (!devs) 36 + return ERR_PTR(-ENOMEM); 37 + devs[0] = sb->s_bdev; 38 + *num_devs = 1; 39 + return devs; 44 40 } 45 41 46 42 static unsigned int fscrypt_get_dun_bytes(const struct fscrypt_info *ci) ··· 70 74 * helpful for debugging problems where the "wrong" implementation is used. 71 75 */ 72 76 static void fscrypt_log_blk_crypto_impl(struct fscrypt_mode *mode, 73 - struct request_queue **devs, 74 - int num_devs, 77 + struct block_device **devs, 78 + unsigned int num_devs, 75 79 const struct blk_crypto_config *cfg) 76 80 { 77 - int i; 81 + unsigned int i; 78 82 79 83 for (i = 0; i < num_devs; i++) { 84 + struct request_queue *q = bdev_get_queue(devs[i]); 85 + 80 86 if (!IS_ENABLED(CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK) || 81 - __blk_crypto_cfg_supported(devs[i]->crypto_profile, cfg)) { 87 + __blk_crypto_cfg_supported(q->crypto_profile, cfg)) { 82 88 if (!xchg(&mode->logged_blk_crypto_native, 1)) 83 89 pr_info("fscrypt: %s using blk-crypto (native)\n", 84 90 mode->friendly_name); ··· 97 99 const struct inode *inode = ci->ci_inode; 98 100 struct super_block *sb = inode->i_sb; 99 101 struct blk_crypto_config crypto_cfg; 100 - int num_devs; 101 - struct request_queue **devs; 102 - int i; 102 + struct block_device **devs; 103 + unsigned int num_devs; 104 + unsigned int i; 103 105 104 106 /* The file must need contents encryption, not filenames encryption */ 105 107 if (!S_ISREG(inode->i_mode)) ··· 127 129 return 0; 128 130 129 131 /* 130 - * On all the filesystem's devices, blk-crypto must support the crypto 131 - * configuration that the file would use. 132 + * On all the filesystem's block devices, blk-crypto must support the 133 + * crypto configuration that the file would use. 132 134 */ 133 135 crypto_cfg.crypto_mode = ci->ci_mode->blk_crypto_mode; 134 136 crypto_cfg.data_unit_size = sb->s_blocksize; 135 137 crypto_cfg.dun_bytes = fscrypt_get_dun_bytes(ci); 136 - num_devs = fscrypt_get_num_devices(sb); 137 - devs = kmalloc_array(num_devs, sizeof(*devs), GFP_KERNEL); 138 - if (!devs) 139 - return -ENOMEM; 140 - fscrypt_get_devices(sb, num_devs, devs); 138 + 139 + devs = fscrypt_get_devices(sb, &num_devs); 140 + if (IS_ERR(devs)) 141 + return PTR_ERR(devs); 141 142 142 143 for (i = 0; i < num_devs; i++) { 143 - if (!blk_crypto_config_supported(devs[i], &crypto_cfg)) 144 + if (!blk_crypto_config_supported(bdev_get_queue(devs[i]), 145 + &crypto_cfg)) 144 146 goto out_free_devs; 145 147 } 146 148 ··· 160 162 const struct inode *inode = ci->ci_inode; 161 163 struct super_block *sb = inode->i_sb; 162 164 enum blk_crypto_mode_num crypto_mode = ci->ci_mode->blk_crypto_mode; 163 - int num_devs = fscrypt_get_num_devices(sb); 164 - int queue_refs = 0; 165 - struct fscrypt_blk_crypto_key *blk_key; 165 + struct blk_crypto_key *blk_key; 166 + struct block_device **devs; 167 + unsigned int num_devs; 168 + unsigned int i; 166 169 int err; 167 - int i; 168 170 169 - blk_key = kzalloc(struct_size(blk_key, devs, num_devs), GFP_KERNEL); 171 + blk_key = kmalloc(sizeof(*blk_key), GFP_KERNEL); 170 172 if (!blk_key) 171 173 return -ENOMEM; 172 174 173 - blk_key->num_devs = num_devs; 174 - fscrypt_get_devices(sb, num_devs, blk_key->devs); 175 - 176 - err = blk_crypto_init_key(&blk_key->base, raw_key, crypto_mode, 175 + err = blk_crypto_init_key(blk_key, raw_key, crypto_mode, 177 176 fscrypt_get_dun_bytes(ci), sb->s_blocksize); 178 177 if (err) { 179 178 fscrypt_err(inode, "error %d initializing blk-crypto key", err); 180 179 goto fail; 181 180 } 182 181 183 - /* 184 - * We have to start using blk-crypto on all the filesystem's devices. 185 - * We also have to save all the request_queue's for later so that the 186 - * key can be evicted from them. This is needed because some keys 187 - * aren't destroyed until after the filesystem was already unmounted 188 - * (namely, the per-mode keys in struct fscrypt_master_key). 189 - */ 190 - for (i = 0; i < num_devs; i++) { 191 - if (!blk_get_queue(blk_key->devs[i])) { 192 - fscrypt_err(inode, "couldn't get request_queue"); 193 - err = -EAGAIN; 194 - goto fail; 195 - } 196 - queue_refs++; 197 - 198 - err = blk_crypto_start_using_key(&blk_key->base, 199 - blk_key->devs[i]); 200 - if (err) { 201 - fscrypt_err(inode, 202 - "error %d starting to use blk-crypto", err); 203 - goto fail; 204 - } 182 + /* Start using blk-crypto on all the filesystem's block devices. */ 183 + devs = fscrypt_get_devices(sb, &num_devs); 184 + if (IS_ERR(devs)) { 185 + err = PTR_ERR(devs); 186 + goto fail; 205 187 } 188 + for (i = 0; i < num_devs; i++) { 189 + err = blk_crypto_start_using_key(blk_key, 190 + bdev_get_queue(devs[i])); 191 + if (err) 192 + break; 193 + } 194 + kfree(devs); 195 + if (err) { 196 + fscrypt_err(inode, "error %d starting to use blk-crypto", err); 197 + goto fail; 198 + } 199 + 206 200 /* 207 201 * Pairs with the smp_load_acquire() in fscrypt_is_key_prepared(). 208 202 * I.e., here we publish ->blk_key with a RELEASE barrier so that ··· 205 215 return 0; 206 216 207 217 fail: 208 - for (i = 0; i < queue_refs; i++) 209 - blk_put_queue(blk_key->devs[i]); 210 218 kfree_sensitive(blk_key); 211 219 return err; 212 220 } 213 221 214 - void fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key) 222 + void fscrypt_destroy_inline_crypt_key(struct super_block *sb, 223 + struct fscrypt_prepared_key *prep_key) 215 224 { 216 - struct fscrypt_blk_crypto_key *blk_key = prep_key->blk_key; 217 - int i; 225 + struct blk_crypto_key *blk_key = prep_key->blk_key; 226 + struct block_device **devs; 227 + unsigned int num_devs; 228 + unsigned int i; 218 229 219 - if (blk_key) { 220 - for (i = 0; i < blk_key->num_devs; i++) { 221 - blk_crypto_evict_key(blk_key->devs[i], &blk_key->base); 222 - blk_put_queue(blk_key->devs[i]); 223 - } 224 - kfree_sensitive(blk_key); 230 + if (!blk_key) 231 + return; 232 + 233 + /* Evict the key from all the filesystem's block devices. */ 234 + devs = fscrypt_get_devices(sb, &num_devs); 235 + if (!IS_ERR(devs)) { 236 + for (i = 0; i < num_devs; i++) 237 + blk_crypto_evict_key(bdev_get_queue(devs[i]), blk_key); 238 + kfree(devs); 225 239 } 240 + kfree_sensitive(blk_key); 226 241 } 227 242 228 243 bool __fscrypt_inode_uses_inline_crypto(const struct inode *inode) ··· 277 282 ci = inode->i_crypt_info; 278 283 279 284 fscrypt_generate_dun(ci, first_lblk, dun); 280 - bio_crypt_set_ctx(bio, &ci->ci_enc_key.blk_key->base, dun, gfp_mask); 285 + bio_crypt_set_ctx(bio, ci->ci_enc_key.blk_key, dun, gfp_mask); 281 286 } 282 287 EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx); 283 288 ··· 364 369 * uses the same pointer. I.e., there's currently no need to support 365 370 * merging requests where the keys are the same but the pointers differ. 366 371 */ 367 - if (bc->bc_key != &inode->i_crypt_info->ci_enc_key.blk_key->base) 372 + if (bc->bc_key != inode->i_crypt_info->ci_enc_key.blk_key) 368 373 return false; 369 374 370 375 fscrypt_generate_dun(inode->i_crypt_info, next_lblk, next_dun);

+268 -229

fs/crypto/keyring.c

··· 18 18 * information about these ioctls. 19 19 */ 20 20 21 + #include <asm/unaligned.h> 21 22 #include <crypto/skcipher.h> 22 23 #include <linux/key-type.h> 23 24 #include <linux/random.h> 24 25 #include <linux/seq_file.h> 25 26 26 27 #include "fscrypt_private.h" 28 + 29 + /* The master encryption keys for a filesystem (->s_master_keys) */ 30 + struct fscrypt_keyring { 31 + /* 32 + * Lock that protects ->key_hashtable. It does *not* protect the 33 + * fscrypt_master_key structs themselves. 34 + */ 35 + spinlock_t lock; 36 + 37 + /* Hash table that maps fscrypt_key_specifier to fscrypt_master_key */ 38 + struct hlist_head key_hashtable[128]; 39 + }; 27 40 28 41 static void wipe_master_key_secret(struct fscrypt_master_key_secret *secret) 29 42 { ··· 51 38 memzero_explicit(src, sizeof(*src)); 52 39 } 53 40 54 - static void free_master_key(struct fscrypt_master_key *mk) 41 + static void fscrypt_free_master_key(struct rcu_head *head) 55 42 { 43 + struct fscrypt_master_key *mk = 44 + container_of(head, struct fscrypt_master_key, mk_rcu_head); 45 + /* 46 + * The master key secret and any embedded subkeys should have already 47 + * been wiped when the last active reference to the fscrypt_master_key 48 + * struct was dropped; doing it here would be unnecessarily late. 49 + * Nevertheless, use kfree_sensitive() in case anything was missed. 50 + */ 51 + kfree_sensitive(mk); 52 + } 53 + 54 + void fscrypt_put_master_key(struct fscrypt_master_key *mk) 55 + { 56 + if (!refcount_dec_and_test(&mk->mk_struct_refs)) 57 + return; 58 + /* 59 + * No structural references left, so free ->mk_users, and also free the 60 + * fscrypt_master_key struct itself after an RCU grace period ensures 61 + * that concurrent keyring lookups can no longer find it. 62 + */ 63 + WARN_ON(refcount_read(&mk->mk_active_refs) != 0); 64 + key_put(mk->mk_users); 65 + mk->mk_users = NULL; 66 + call_rcu(&mk->mk_rcu_head, fscrypt_free_master_key); 67 + } 68 + 69 + void fscrypt_put_master_key_activeref(struct fscrypt_master_key *mk) 70 + { 71 + struct super_block *sb = mk->mk_sb; 72 + struct fscrypt_keyring *keyring = sb->s_master_keys; 56 73 size_t i; 57 74 58 - wipe_master_key_secret(&mk->mk_secret); 75 + if (!refcount_dec_and_test(&mk->mk_active_refs)) 76 + return; 77 + /* 78 + * No active references left, so complete the full removal of this 79 + * fscrypt_master_key struct by removing it from the keyring and 80 + * destroying any subkeys embedded in it. 81 + */ 82 + 83 + spin_lock(&keyring->lock); 84 + hlist_del_rcu(&mk->mk_node); 85 + spin_unlock(&keyring->lock); 86 + 87 + /* 88 + * ->mk_active_refs == 0 implies that ->mk_secret is not present and 89 + * that ->mk_decrypted_inodes is empty. 90 + */ 91 + WARN_ON(is_master_key_secret_present(&mk->mk_secret)); 92 + WARN_ON(!list_empty(&mk->mk_decrypted_inodes)); 59 93 60 94 for (i = 0; i <= FSCRYPT_MODE_MAX; i++) { 61 - fscrypt_destroy_prepared_key(&mk->mk_direct_keys[i]); 62 - fscrypt_destroy_prepared_key(&mk->mk_iv_ino_lblk_64_keys[i]); 63 - fscrypt_destroy_prepared_key(&mk->mk_iv_ino_lblk_32_keys[i]); 95 + fscrypt_destroy_prepared_key( 96 + sb, &mk->mk_direct_keys[i]); 97 + fscrypt_destroy_prepared_key( 98 + sb, &mk->mk_iv_ino_lblk_64_keys[i]); 99 + fscrypt_destroy_prepared_key( 100 + sb, &mk->mk_iv_ino_lblk_32_keys[i]); 64 101 } 102 + memzero_explicit(&mk->mk_ino_hash_key, 103 + sizeof(mk->mk_ino_hash_key)); 104 + mk->mk_ino_hash_key_initialized = false; 65 105 66 - key_put(mk->mk_users); 67 - kfree_sensitive(mk); 106 + /* Drop the structural ref associated with the active refs. */ 107 + fscrypt_put_master_key(mk); 68 108 } 69 109 70 110 static inline bool valid_key_spec(const struct fscrypt_key_specifier *spec) ··· 126 60 return false; 127 61 return master_key_spec_len(spec) != 0; 128 62 } 129 - 130 - static int fscrypt_key_instantiate(struct key *key, 131 - struct key_preparsed_payload *prep) 132 - { 133 - key->payload.data[0] = (struct fscrypt_master_key *)prep->data; 134 - return 0; 135 - } 136 - 137 - static void fscrypt_key_destroy(struct key *key) 138 - { 139 - free_master_key(key->payload.data[0]); 140 - } 141 - 142 - static void fscrypt_key_describe(const struct key *key, struct seq_file *m) 143 - { 144 - seq_puts(m, key->description); 145 - 146 - if (key_is_positive(key)) { 147 - const struct fscrypt_master_key *mk = key->payload.data[0]; 148 - 149 - if (!is_master_key_secret_present(&mk->mk_secret)) 150 - seq_puts(m, ": secret removed"); 151 - } 152 - } 153 - 154 - /* 155 - * Type of key in ->s_master_keys. Each key of this type represents a master 156 - * key which has been added to the filesystem. Its payload is a 157 - * 'struct fscrypt_master_key'. The "." prefix in the key type name prevents 158 - * users from adding keys of this type via the keyrings syscalls rather than via 159 - * the intended method of FS_IOC_ADD_ENCRYPTION_KEY. 160 - */ 161 - static struct key_type key_type_fscrypt = { 162 - .name = "._fscrypt", 163 - .instantiate = fscrypt_key_instantiate, 164 - .destroy = fscrypt_key_destroy, 165 - .describe = fscrypt_key_describe, 166 - }; 167 63 168 64 static int fscrypt_user_key_instantiate(struct key *key, 169 65 struct key_preparsed_payload *prep) ··· 159 131 .describe = fscrypt_user_key_describe, 160 132 }; 161 133 162 - /* Search ->s_master_keys or ->mk_users */ 163 - static struct key *search_fscrypt_keyring(struct key *keyring, 164 - struct key_type *type, 165 - const char *description) 166 - { 167 - /* 168 - * We need to mark the keyring reference as "possessed" so that we 169 - * acquire permission to search it, via the KEY_POS_SEARCH permission. 170 - */ 171 - key_ref_t keyref = make_key_ref(keyring, true /* possessed */); 172 - 173 - keyref = keyring_search(keyref, type, description, false); 174 - if (IS_ERR(keyref)) { 175 - if (PTR_ERR(keyref) == -EAGAIN || /* not found */ 176 - PTR_ERR(keyref) == -EKEYREVOKED) /* recently invalidated */ 177 - keyref = ERR_PTR(-ENOKEY); 178 - return ERR_CAST(keyref); 179 - } 180 - return key_ref_to_ptr(keyref); 181 - } 182 - 183 - #define FSCRYPT_FS_KEYRING_DESCRIPTION_SIZE \ 184 - (CONST_STRLEN("fscrypt-") + sizeof_field(struct super_block, s_id)) 185 - 186 - #define FSCRYPT_MK_DESCRIPTION_SIZE (2 * FSCRYPT_KEY_IDENTIFIER_SIZE + 1) 187 - 188 134 #define FSCRYPT_MK_USERS_DESCRIPTION_SIZE \ 189 135 (CONST_STRLEN("fscrypt-") + 2 * FSCRYPT_KEY_IDENTIFIER_SIZE + \ 190 136 CONST_STRLEN("-users") + 1) 191 137 192 138 #define FSCRYPT_MK_USER_DESCRIPTION_SIZE \ 193 139 (2 * FSCRYPT_KEY_IDENTIFIER_SIZE + CONST_STRLEN(".uid.") + 10 + 1) 194 - 195 - static void format_fs_keyring_description( 196 - char description[FSCRYPT_FS_KEYRING_DESCRIPTION_SIZE], 197 - const struct super_block *sb) 198 - { 199 - sprintf(description, "fscrypt-%s", sb->s_id); 200 - } 201 - 202 - static void format_mk_description( 203 - char description[FSCRYPT_MK_DESCRIPTION_SIZE], 204 - const struct fscrypt_key_specifier *mk_spec) 205 - { 206 - sprintf(description, "%*phN", 207 - master_key_spec_len(mk_spec), (u8 *)&mk_spec->u); 208 - } 209 140 210 141 static void format_mk_users_keyring_description( 211 142 char description[FSCRYPT_MK_USERS_DESCRIPTION_SIZE], ··· 186 199 /* Create ->s_master_keys if needed. Synchronized by fscrypt_add_key_mutex. */ 187 200 static int allocate_filesystem_keyring(struct super_block *sb) 188 201 { 189 - char description[FSCRYPT_FS_KEYRING_DESCRIPTION_SIZE]; 190 - struct key *keyring; 202 + struct fscrypt_keyring *keyring; 191 203 192 204 if (sb->s_master_keys) 193 205 return 0; 194 206 195 - format_fs_keyring_description(description, sb); 196 - keyring = keyring_alloc(description, GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, 197 - current_cred(), KEY_POS_SEARCH | 198 - KEY_USR_SEARCH | KEY_USR_READ | KEY_USR_VIEW, 199 - KEY_ALLOC_NOT_IN_QUOTA, NULL, NULL); 200 - if (IS_ERR(keyring)) 201 - return PTR_ERR(keyring); 202 - 207 + keyring = kzalloc(sizeof(*keyring), GFP_KERNEL); 208 + if (!keyring) 209 + return -ENOMEM; 210 + spin_lock_init(&keyring->lock); 203 211 /* 204 212 * Pairs with the smp_load_acquire() in fscrypt_find_master_key(). 205 213 * I.e., here we publish ->s_master_keys with a RELEASE barrier so that ··· 204 222 return 0; 205 223 } 206 224 207 - void fscrypt_sb_free(struct super_block *sb) 225 + /* 226 + * This is called at unmount time to release all encryption keys that have been 227 + * added to the filesystem, along with the keyring that contains them. 228 + * 229 + * Note that besides clearing and freeing memory, this might need to evict keys 230 + * from the keyslots of an inline crypto engine. Therefore, this must be called 231 + * while the filesystem's underlying block device(s) are still available. 232 + */ 233 + void fscrypt_sb_delete(struct super_block *sb) 208 234 { 209 - key_put(sb->s_master_keys); 235 + struct fscrypt_keyring *keyring = sb->s_master_keys; 236 + size_t i; 237 + 238 + if (!keyring) 239 + return; 240 + 241 + for (i = 0; i < ARRAY_SIZE(keyring->key_hashtable); i++) { 242 + struct hlist_head *bucket = &keyring->key_hashtable[i]; 243 + struct fscrypt_master_key *mk; 244 + struct hlist_node *tmp; 245 + 246 + hlist_for_each_entry_safe(mk, tmp, bucket, mk_node) { 247 + /* 248 + * Since all inodes were already evicted, every key 249 + * remaining in the keyring should have an empty inode 250 + * list, and should only still be in the keyring due to 251 + * the single active ref associated with ->mk_secret. 252 + * There should be no structural refs beyond the one 253 + * associated with the active ref. 254 + */ 255 + WARN_ON(refcount_read(&mk->mk_active_refs) != 1); 256 + WARN_ON(refcount_read(&mk->mk_struct_refs) != 1); 257 + WARN_ON(!is_master_key_secret_present(&mk->mk_secret)); 258 + wipe_master_key_secret(&mk->mk_secret); 259 + fscrypt_put_master_key_activeref(mk); 260 + } 261 + } 262 + kfree_sensitive(keyring); 210 263 sb->s_master_keys = NULL; 211 264 } 212 265 213 - /* 214 - * Find the specified master key in ->s_master_keys. 215 - * Returns ERR_PTR(-ENOKEY) if not found. 216 - */ 217 - struct key *fscrypt_find_master_key(struct super_block *sb, 218 - const struct fscrypt_key_specifier *mk_spec) 266 + static struct hlist_head * 267 + fscrypt_mk_hash_bucket(struct fscrypt_keyring *keyring, 268 + const struct fscrypt_key_specifier *mk_spec) 219 269 { 220 - struct key *keyring; 221 - char description[FSCRYPT_MK_DESCRIPTION_SIZE]; 270 + /* 271 + * Since key specifiers should be "random" values, it is sufficient to 272 + * use a trivial hash function that just takes the first several bits of 273 + * the key specifier. 274 + */ 275 + unsigned long i = get_unaligned((unsigned long *)&mk_spec->u); 276 + 277 + return &keyring->key_hashtable[i % ARRAY_SIZE(keyring->key_hashtable)]; 278 + } 279 + 280 + /* 281 + * Find the specified master key struct in ->s_master_keys and take a structural 282 + * ref to it. The structural ref guarantees that the key struct continues to 283 + * exist, but it does *not* guarantee that ->s_master_keys continues to contain 284 + * the key struct. The structural ref needs to be dropped by 285 + * fscrypt_put_master_key(). Returns NULL if the key struct is not found. 286 + */ 287 + struct fscrypt_master_key * 288 + fscrypt_find_master_key(struct super_block *sb, 289 + const struct fscrypt_key_specifier *mk_spec) 290 + { 291 + struct fscrypt_keyring *keyring; 292 + struct hlist_head *bucket; 293 + struct fscrypt_master_key *mk; 222 294 223 295 /* 224 296 * Pairs with the smp_store_release() in allocate_filesystem_keyring(). ··· 282 246 */ 283 247 keyring = smp_load_acquire(&sb->s_master_keys); 284 248 if (keyring == NULL) 285 - return ERR_PTR(-ENOKEY); /* No keyring yet, so no keys yet. */ 249 + return NULL; /* No keyring yet, so no keys yet. */ 286 250 287 - format_mk_description(description, mk_spec); 288 - return search_fscrypt_keyring(keyring, &key_type_fscrypt, description); 251 + bucket = fscrypt_mk_hash_bucket(keyring, mk_spec); 252 + rcu_read_lock(); 253 + switch (mk_spec->type) { 254 + case FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR: 255 + hlist_for_each_entry_rcu(mk, bucket, mk_node) { 256 + if (mk->mk_spec.type == 257 + FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR && 258 + memcmp(mk->mk_spec.u.descriptor, 259 + mk_spec->u.descriptor, 260 + FSCRYPT_KEY_DESCRIPTOR_SIZE) == 0 && 261 + refcount_inc_not_zero(&mk->mk_struct_refs)) 262 + goto out; 263 + } 264 + break; 265 + case FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER: 266 + hlist_for_each_entry_rcu(mk, bucket, mk_node) { 267 + if (mk->mk_spec.type == 268 + FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER && 269 + memcmp(mk->mk_spec.u.identifier, 270 + mk_spec->u.identifier, 271 + FSCRYPT_KEY_IDENTIFIER_SIZE) == 0 && 272 + refcount_inc_not_zero(&mk->mk_struct_refs)) 273 + goto out; 274 + } 275 + break; 276 + } 277 + mk = NULL; 278 + out: 279 + rcu_read_unlock(); 280 + return mk; 289 281 } 290 282 291 283 static int allocate_master_key_users_keyring(struct fscrypt_master_key *mk) ··· 341 277 static struct key *find_master_key_user(struct fscrypt_master_key *mk) 342 278 { 343 279 char description[FSCRYPT_MK_USER_DESCRIPTION_SIZE]; 280 + key_ref_t keyref; 344 281 345 282 format_mk_user_description(description, mk->mk_spec.u.identifier); 346 - return search_fscrypt_keyring(mk->mk_users, &key_type_fscrypt_user, 347 - description); 283 + 284 + /* 285 + * We need to mark the keyring reference as "possessed" so that we 286 + * acquire permission to search it, via the KEY_POS_SEARCH permission. 287 + */ 288 + keyref = keyring_search(make_key_ref(mk->mk_users, true /*possessed*/), 289 + &key_type_fscrypt_user, description, false); 290 + if (IS_ERR(keyref)) { 291 + if (PTR_ERR(keyref) == -EAGAIN || /* not found */ 292 + PTR_ERR(keyref) == -EKEYREVOKED) /* recently invalidated */ 293 + keyref = ERR_PTR(-ENOKEY); 294 + return ERR_CAST(keyref); 295 + } 296 + return key_ref_to_ptr(keyref); 348 297 } 349 298 350 299 /* 351 300 * Give the current user a "key" in ->mk_users. This charges the user's quota 352 301 * and marks the master key as added by the current user, so that it cannot be 353 - * removed by another user with the key. Either the master key's key->sem must 354 - * be held for write, or the master key must be still undergoing initialization. 302 + * removed by another user with the key. Either ->mk_sem must be held for 303 + * write, or the master key must be still undergoing initialization. 355 304 */ 356 305 static int add_master_key_user(struct fscrypt_master_key *mk) 357 306 { ··· 386 309 387 310 /* 388 311 * Remove the current user's "key" from ->mk_users. 389 - * The master key's key->sem must be held for write. 312 + * ->mk_sem must be held for write. 390 313 * 391 314 * Returns 0 if removed, -ENOKEY if not found, or another -errno code. 392 315 */ ··· 404 327 } 405 328 406 329 /* 407 - * Allocate a new fscrypt_master_key which contains the given secret, set it as 408 - * the payload of a new 'struct key' of type fscrypt, and link the 'struct key' 409 - * into the given keyring. Synchronized by fscrypt_add_key_mutex. 330 + * Allocate a new fscrypt_master_key, transfer the given secret over to it, and 331 + * insert it into sb->s_master_keys. 410 332 */ 411 - static int add_new_master_key(struct fscrypt_master_key_secret *secret, 412 - const struct fscrypt_key_specifier *mk_spec, 413 - struct key *keyring) 333 + static int add_new_master_key(struct super_block *sb, 334 + struct fscrypt_master_key_secret *secret, 335 + const struct fscrypt_key_specifier *mk_spec) 414 336 { 337 + struct fscrypt_keyring *keyring = sb->s_master_keys; 415 338 struct fscrypt_master_key *mk; 416 - char description[FSCRYPT_MK_DESCRIPTION_SIZE]; 417 - struct key *key; 418 339 int err; 419 340 420 341 mk = kzalloc(sizeof(*mk), GFP_KERNEL); 421 342 if (!mk) 422 343 return -ENOMEM; 423 344 345 + mk->mk_sb = sb; 346 + init_rwsem(&mk->mk_sem); 347 + refcount_set(&mk->mk_struct_refs, 1); 424 348 mk->mk_spec = *mk_spec; 425 349 426 - move_master_key_secret(&mk->mk_secret, secret); 427 - 428 - refcount_set(&mk->mk_refcount, 1); /* secret is present */ 429 350 INIT_LIST_HEAD(&mk->mk_decrypted_inodes); 430 351 spin_lock_init(&mk->mk_decrypted_inodes_lock); 431 352 432 353 if (mk_spec->type == FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER) { 433 354 err = allocate_master_key_users_keyring(mk); 434 355 if (err) 435 - goto out_free_mk; 356 + goto out_put; 436 357 err = add_master_key_user(mk); 437 358 if (err) 438 - goto out_free_mk; 359 + goto out_put; 439 360 } 440 361 441 - /* 442 - * Note that we don't charge this key to anyone's quota, since when 443 - * ->mk_users is in use those keys are charged instead, and otherwise 444 - * (when ->mk_users isn't in use) only root can add these keys. 445 - */ 446 - format_mk_description(description, mk_spec); 447 - key = key_alloc(&key_type_fscrypt, description, 448 - GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, current_cred(), 449 - KEY_POS_SEARCH | KEY_USR_SEARCH | KEY_USR_VIEW, 450 - KEY_ALLOC_NOT_IN_QUOTA, NULL); 451 - if (IS_ERR(key)) { 452 - err = PTR_ERR(key); 453 - goto out_free_mk; 454 - } 455 - err = key_instantiate_and_link(key, mk, sizeof(*mk), keyring, NULL); 456 - key_put(key); 457 - if (err) 458 - goto out_free_mk; 362 + move_master_key_secret(&mk->mk_secret, secret); 363 + refcount_set(&mk->mk_active_refs, 1); /* ->mk_secret is present */ 459 364 365 + spin_lock(&keyring->lock); 366 + hlist_add_head_rcu(&mk->mk_node, 367 + fscrypt_mk_hash_bucket(keyring, mk_spec)); 368 + spin_unlock(&keyring->lock); 460 369 return 0; 461 370 462 - out_free_mk: 463 - free_master_key(mk); 371 + out_put: 372 + fscrypt_put_master_key(mk); 464 373 return err; 465 374 } 466 375 ··· 455 392 static int add_existing_master_key(struct fscrypt_master_key *mk, 456 393 struct fscrypt_master_key_secret *secret) 457 394 { 458 - struct key *mk_user; 459 - bool rekey; 460 395 int err; 461 396 462 397 /* 463 398 * If the current user is already in ->mk_users, then there's nothing to 464 - * do. (Not applicable for v1 policy keys, which have NULL ->mk_users.) 399 + * do. Otherwise, we need to add the user to ->mk_users. (Neither is 400 + * applicable for v1 policy keys, which have NULL ->mk_users.) 465 401 */ 466 402 if (mk->mk_users) { 467 - mk_user = find_master_key_user(mk); 403 + struct key *mk_user = find_master_key_user(mk); 404 + 468 405 if (mk_user != ERR_PTR(-ENOKEY)) { 469 406 if (IS_ERR(mk_user)) 470 407 return PTR_ERR(mk_user); 471 408 key_put(mk_user); 472 409 return 0; 473 410 } 474 - } 475 - 476 - /* If we'll be re-adding ->mk_secret, try to take the reference. */ 477 - rekey = !is_master_key_secret_present(&mk->mk_secret); 478 - if (rekey && !refcount_inc_not_zero(&mk->mk_refcount)) 479 - return KEY_DEAD; 480 - 481 - /* Add the current user to ->mk_users, if applicable. */ 482 - if (mk->mk_users) { 483 411 err = add_master_key_user(mk); 484 - if (err) { 485 - if (rekey && refcount_dec_and_test(&mk->mk_refcount)) 486 - return KEY_DEAD; 412 + if (err) 487 413 return err; 488 - } 489 414 } 490 415 491 416 /* Re-add the secret if needed. */ 492 - if (rekey) 417 + if (!is_master_key_secret_present(&mk->mk_secret)) { 418 + if (!refcount_inc_not_zero(&mk->mk_active_refs)) 419 + return KEY_DEAD; 493 420 move_master_key_secret(&mk->mk_secret, secret); 421 + } 422 + 494 423 return 0; 495 424 } 496 425 ··· 491 436 const struct fscrypt_key_specifier *mk_spec) 492 437 { 493 438 static DEFINE_MUTEX(fscrypt_add_key_mutex); 494 - struct key *key; 439 + struct fscrypt_master_key *mk; 495 440 int err; 496 441 497 442 mutex_lock(&fscrypt_add_key_mutex); /* serialize find + link */ 498 - retry: 499 - key = fscrypt_find_master_key(sb, mk_spec); 500 - if (IS_ERR(key)) { 501 - err = PTR_ERR(key); 502 - if (err != -ENOKEY) 503 - goto out_unlock; 443 + 444 + mk = fscrypt_find_master_key(sb, mk_spec); 445 + if (!mk) { 504 446 /* Didn't find the key in ->s_master_keys. Add it. */ 505 447 err = allocate_filesystem_keyring(sb); 506 - if (err) 507 - goto out_unlock; 508 - err = add_new_master_key(secret, mk_spec, sb->s_master_keys); 448 + if (!err) 449 + err = add_new_master_key(sb, secret, mk_spec); 509 450 } else { 510 451 /* 511 452 * Found the key in ->s_master_keys. Re-add the secret if 512 453 * needed, and add the user to ->mk_users if needed. 513 454 */ 514 - down_write(&key->sem); 515 - err = add_existing_master_key(key->payload.data[0], secret); 516 - up_write(&key->sem); 455 + down_write(&mk->mk_sem); 456 + err = add_existing_master_key(mk, secret); 457 + up_write(&mk->mk_sem); 517 458 if (err == KEY_DEAD) { 518 - /* Key being removed or needs to be removed */ 519 - key_invalidate(key); 520 - key_put(key); 521 - goto retry; 459 + /* 460 + * We found a key struct, but it's already been fully 461 + * removed. Ignore the old struct and add a new one. 462 + * fscrypt_add_key_mutex means we don't need to worry 463 + * about concurrent adds. 464 + */ 465 + err = add_new_master_key(sb, secret, mk_spec); 522 466 } 523 - key_put(key); 467 + fscrypt_put_master_key(mk); 524 468 } 525 - out_unlock: 526 469 mutex_unlock(&fscrypt_add_key_mutex); 527 470 return err; 528 471 } ··· 824 771 const u8 identifier[FSCRYPT_KEY_IDENTIFIER_SIZE]) 825 772 { 826 773 struct fscrypt_key_specifier mk_spec; 827 - struct key *key, *mk_user; 828 774 struct fscrypt_master_key *mk; 775 + struct key *mk_user; 829 776 int err; 830 777 831 778 mk_spec.type = FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER; 832 779 memcpy(mk_spec.u.identifier, identifier, FSCRYPT_KEY_IDENTIFIER_SIZE); 833 780 834 - key = fscrypt_find_master_key(sb, &mk_spec); 835 - if (IS_ERR(key)) { 836 - err = PTR_ERR(key); 781 + mk = fscrypt_find_master_key(sb, &mk_spec); 782 + if (!mk) { 783 + err = -ENOKEY; 837 784 goto out; 838 785 } 839 - mk = key->payload.data[0]; 786 + down_read(&mk->mk_sem); 840 787 mk_user = find_master_key_user(mk); 841 788 if (IS_ERR(mk_user)) { 842 789 err = PTR_ERR(mk_user); ··· 844 791 key_put(mk_user); 845 792 err = 0; 846 793 } 847 - key_put(key); 794 + up_read(&mk->mk_sem); 795 + fscrypt_put_master_key(mk); 848 796 out: 849 797 if (err == -ENOKEY && capable(CAP_FOWNER)) 850 798 err = 0; ··· 1007 953 struct super_block *sb = file_inode(filp)->i_sb; 1008 954 struct fscrypt_remove_key_arg __user *uarg = _uarg; 1009 955 struct fscrypt_remove_key_arg arg; 1010 - struct key *key; 1011 956 struct fscrypt_master_key *mk; 1012 957 u32 status_flags = 0; 1013 958 int err; 1014 - bool dead; 959 + bool inodes_remain; 1015 960 1016 961 if (copy_from_user(&arg, uarg, sizeof(arg))) 1017 962 return -EFAULT; ··· 1030 977 return -EACCES; 1031 978 1032 979 /* Find the key being removed. */ 1033 - key = fscrypt_find_master_key(sb, &arg.key_spec); 1034 - if (IS_ERR(key)) 1035 - return PTR_ERR(key); 1036 - mk = key->payload.data[0]; 1037 - 1038 - down_write(&key->sem); 980 + mk = fscrypt_find_master_key(sb, &arg.key_spec); 981 + if (!mk) 982 + return -ENOKEY; 983 + down_write(&mk->mk_sem); 1039 984 1040 985 /* If relevant, remove current user's (or all users) claim to the key */ 1041 986 if (mk->mk_users && mk->mk_users->keys.nr_leaves_on_tree != 0) { ··· 1042 991 else 1043 992 err = remove_master_key_user(mk); 1044 993 if (err) { 1045 - up_write(&key->sem); 994 + up_write(&mk->mk_sem); 1046 995 goto out_put_key; 1047 996 } 1048 997 if (mk->mk_users->keys.nr_leaves_on_tree != 0) { ··· 1054 1003 status_flags |= 1055 1004 FSCRYPT_KEY_REMOVAL_STATUS_FLAG_OTHER_USERS; 1056 1005 err = 0; 1057 - up_write(&key->sem); 1006 + up_write(&mk->mk_sem); 1058 1007 goto out_put_key; 1059 1008 } 1060 1009 } 1061 1010 1062 1011 /* No user claims remaining. Go ahead and wipe the secret. */ 1063 - dead = false; 1012 + err = -ENOKEY; 1064 1013 if (is_master_key_secret_present(&mk->mk_secret)) { 1065 1014 wipe_master_key_secret(&mk->mk_secret); 1066 - dead = refcount_dec_and_test(&mk->mk_refcount); 1067 - } 1068 - up_write(&key->sem); 1069 - if (dead) { 1070 - /* 1071 - * No inodes reference the key, and we wiped the secret, so the 1072 - * key object is free to be removed from the keyring. 1073 - */ 1074 - key_invalidate(key); 1015 + fscrypt_put_master_key_activeref(mk); 1075 1016 err = 0; 1076 - } else { 1017 + } 1018 + inodes_remain = refcount_read(&mk->mk_active_refs) > 0; 1019 + up_write(&mk->mk_sem); 1020 + 1021 + if (inodes_remain) { 1077 1022 /* Some inodes still reference this key; try to evict them. */ 1078 1023 err = try_to_lock_encrypted_files(sb, mk); 1079 1024 if (err == -EBUSY) { ··· 1085 1038 * has been fully removed including all files locked. 1086 1039 */ 1087 1040 out_put_key: 1088 - key_put(key); 1041 + fscrypt_put_master_key(mk); 1089 1042 if (err == 0) 1090 1043 err = put_user(status_flags, &uarg->removal_status_flags); 1091 1044 return err; ··· 1132 1085 { 1133 1086 struct super_block *sb = file_inode(filp)->i_sb; 1134 1087 struct fscrypt_get_key_status_arg arg; 1135 - struct key *key; 1136 1088 struct fscrypt_master_key *mk; 1137 1089 int err; 1138 1090 ··· 1148 1102 arg.user_count = 0; 1149 1103 memset(arg.__out_reserved, 0, sizeof(arg.__out_reserved)); 1150 1104 1151 - key = fscrypt_find_master_key(sb, &arg.key_spec); 1152 - if (IS_ERR(key)) { 1153 - if (key != ERR_PTR(-ENOKEY)) 1154 - return PTR_ERR(key); 1105 + mk = fscrypt_find_master_key(sb, &arg.key_spec); 1106 + if (!mk) { 1155 1107 arg.status = FSCRYPT_KEY_STATUS_ABSENT; 1156 1108 err = 0; 1157 1109 goto out; 1158 1110 } 1159 - mk = key->payload.data[0]; 1160 - down_read(&key->sem); 1111 + down_read(&mk->mk_sem); 1161 1112 1162 1113 if (!is_master_key_secret_present(&mk->mk_secret)) { 1163 - arg.status = FSCRYPT_KEY_STATUS_INCOMPLETELY_REMOVED; 1114 + arg.status = refcount_read(&mk->mk_active_refs) > 0 ? 1115 + FSCRYPT_KEY_STATUS_INCOMPLETELY_REMOVED : 1116 + FSCRYPT_KEY_STATUS_ABSENT /* raced with full removal */; 1164 1117 err = 0; 1165 1118 goto out_release_key; 1166 1119 } ··· 1181 1136 } 1182 1137 err = 0; 1183 1138 out_release_key: 1184 - up_read(&key->sem); 1185 - key_put(key); 1139 + up_read(&mk->mk_sem); 1140 + fscrypt_put_master_key(mk); 1186 1141 out: 1187 1142 if (!err && copy_to_user(uarg, &arg, sizeof(arg))) 1188 1143 err = -EFAULT; ··· 1194 1149 { 1195 1150 int err; 1196 1151 1197 - err = register_key_type(&key_type_fscrypt); 1198 - if (err) 1199 - return err; 1200 - 1201 1152 err = register_key_type(&key_type_fscrypt_user); 1202 1153 if (err) 1203 - goto err_unregister_fscrypt; 1154 + return err; 1204 1155 1205 1156 err = register_key_type(&key_type_fscrypt_provisioning); 1206 1157 if (err) ··· 1206 1165 1207 1166 err_unregister_fscrypt_user: 1208 1167 unregister_key_type(&key_type_fscrypt_user); 1209 - err_unregister_fscrypt: 1210 - unregister_key_type(&key_type_fscrypt); 1211 1168 return err; 1212 1169 }

+37 -52

fs/crypto/keysetup.c

··· 9 9 */ 10 10 11 11 #include <crypto/skcipher.h> 12 - #include <linux/key.h> 13 12 #include <linux/random.h> 14 13 15 14 #include "fscrypt_private.h" ··· 154 155 } 155 156 156 157 /* Destroy a crypto transform object and/or blk-crypto key. */ 157 - void fscrypt_destroy_prepared_key(struct fscrypt_prepared_key *prep_key) 158 + void fscrypt_destroy_prepared_key(struct super_block *sb, 159 + struct fscrypt_prepared_key *prep_key) 158 160 { 159 161 crypto_free_skcipher(prep_key->tfm); 160 - fscrypt_destroy_inline_crypt_key(prep_key); 162 + fscrypt_destroy_inline_crypt_key(sb, prep_key); 163 + memzero_explicit(prep_key, sizeof(*prep_key)); 161 164 } 162 165 163 166 /* Given a per-file encryption key, set up the file's crypto transform object */ ··· 413 412 /* 414 413 * Find the master key, then set up the inode's actual encryption key. 415 414 * 416 - * If the master key is found in the filesystem-level keyring, then the 417 - * corresponding 'struct key' is returned in *master_key_ret with its semaphore 418 - * read-locked. This is needed to ensure that only one task links the 419 - * fscrypt_info into ->mk_decrypted_inodes (as multiple tasks may race to create 420 - * an fscrypt_info for the same inode), and to synchronize the master key being 421 - * removed with a new inode starting to use it. 415 + * If the master key is found in the filesystem-level keyring, then it is 416 + * returned in *mk_ret with its semaphore read-locked. This is needed to ensure 417 + * that only one task links the fscrypt_info into ->mk_decrypted_inodes (as 418 + * multiple tasks may race to create an fscrypt_info for the same inode), and to 419 + * synchronize the master key being removed with a new inode starting to use it. 422 420 */ 423 421 static int setup_file_encryption_key(struct fscrypt_info *ci, 424 422 bool need_dirhash_key, 425 - struct key **master_key_ret) 423 + struct fscrypt_master_key **mk_ret) 426 424 { 427 - struct key *key; 428 - struct fscrypt_master_key *mk = NULL; 429 425 struct fscrypt_key_specifier mk_spec; 426 + struct fscrypt_master_key *mk; 430 427 int err; 431 428 432 429 err = fscrypt_select_encryption_impl(ci); ··· 435 436 if (err) 436 437 return err; 437 438 438 - key = fscrypt_find_master_key(ci->ci_inode->i_sb, &mk_spec); 439 - if (IS_ERR(key)) { 440 - if (key != ERR_PTR(-ENOKEY) || 441 - ci->ci_policy.version != FSCRYPT_POLICY_V1) 442 - return PTR_ERR(key); 439 + mk = fscrypt_find_master_key(ci->ci_inode->i_sb, &mk_spec); 440 + if (!mk) { 441 + if (ci->ci_policy.version != FSCRYPT_POLICY_V1) 442 + return -ENOKEY; 443 443 444 444 /* 445 445 * As a legacy fallback for v1 policies, search for the key in ··· 448 450 */ 449 451 return fscrypt_setup_v1_file_key_via_subscribed_keyrings(ci); 450 452 } 451 - 452 - mk = key->payload.data[0]; 453 - down_read(&key->sem); 453 + down_read(&mk->mk_sem); 454 454 455 455 /* Has the secret been removed (via FS_IOC_REMOVE_ENCRYPTION_KEY)? */ 456 456 if (!is_master_key_secret_present(&mk->mk_secret)) { ··· 476 480 if (err) 477 481 goto out_release_key; 478 482 479 - *master_key_ret = key; 483 + *mk_ret = mk; 480 484 return 0; 481 485 482 486 out_release_key: 483 - up_read(&key->sem); 484 - key_put(key); 487 + up_read(&mk->mk_sem); 488 + fscrypt_put_master_key(mk); 485 489 return err; 486 490 } 487 491 488 492 static void put_crypt_info(struct fscrypt_info *ci) 489 493 { 490 - struct key *key; 494 + struct fscrypt_master_key *mk; 491 495 492 496 if (!ci) 493 497 return; ··· 495 499 if (ci->ci_direct_key) 496 500 fscrypt_put_direct_key(ci->ci_direct_key); 497 501 else if (ci->ci_owns_key) 498 - fscrypt_destroy_prepared_key(&ci->ci_enc_key); 502 + fscrypt_destroy_prepared_key(ci->ci_inode->i_sb, 503 + &ci->ci_enc_key); 499 504 500 - key = ci->ci_master_key; 501 - if (key) { 502 - struct fscrypt_master_key *mk = key->payload.data[0]; 503 - 505 + mk = ci->ci_master_key; 506 + if (mk) { 504 507 /* 505 508 * Remove this inode from the list of inodes that were unlocked 506 - * with the master key. 507 - * 508 - * In addition, if we're removing the last inode from a key that 509 - * already had its secret removed, invalidate the key so that it 510 - * gets removed from ->s_master_keys. 509 + * with the master key. In addition, if we're removing the last 510 + * inode from a master key struct that already had its secret 511 + * removed, then complete the full removal of the struct. 511 512 */ 512 513 spin_lock(&mk->mk_decrypted_inodes_lock); 513 514 list_del(&ci->ci_master_key_link); 514 515 spin_unlock(&mk->mk_decrypted_inodes_lock); 515 - if (refcount_dec_and_test(&mk->mk_refcount)) 516 - key_invalidate(key); 517 - key_put(key); 516 + fscrypt_put_master_key_activeref(mk); 518 517 } 519 518 memzero_explicit(ci, sizeof(*ci)); 520 519 kmem_cache_free(fscrypt_info_cachep, ci); ··· 523 532 { 524 533 struct fscrypt_info *crypt_info; 525 534 struct fscrypt_mode *mode; 526 - struct key *master_key = NULL; 535 + struct fscrypt_master_key *mk = NULL; 527 536 int res; 528 537 529 538 res = fscrypt_initialize(inode->i_sb->s_cop->flags); ··· 546 555 WARN_ON(mode->ivsize > FSCRYPT_MAX_IV_SIZE); 547 556 crypt_info->ci_mode = mode; 548 557 549 - res = setup_file_encryption_key(crypt_info, need_dirhash_key, 550 - &master_key); 558 + res = setup_file_encryption_key(crypt_info, need_dirhash_key, &mk); 551 559 if (res) 552 560 goto out; 553 561 ··· 561 571 * We won the race and set ->i_crypt_info to our crypt_info. 562 572 * Now link it into the master key's inode list. 563 573 */ 564 - if (master_key) { 565 - struct fscrypt_master_key *mk = 566 - master_key->payload.data[0]; 567 - 568 - refcount_inc(&mk->mk_refcount); 569 - crypt_info->ci_master_key = key_get(master_key); 574 + if (mk) { 575 + crypt_info->ci_master_key = mk; 576 + refcount_inc(&mk->mk_active_refs); 570 577 spin_lock(&mk->mk_decrypted_inodes_lock); 571 578 list_add(&crypt_info->ci_master_key_link, 572 579 &mk->mk_decrypted_inodes); ··· 573 586 } 574 587 res = 0; 575 588 out: 576 - if (master_key) { 577 - up_read(&master_key->sem); 578 - key_put(master_key); 589 + if (mk) { 590 + up_read(&mk->mk_sem); 591 + fscrypt_put_master_key(mk); 579 592 } 580 593 put_crypt_info(crypt_info); 581 594 return res; ··· 740 753 int fscrypt_drop_inode(struct inode *inode) 741 754 { 742 755 const struct fscrypt_info *ci = fscrypt_get_info(inode); 743 - const struct fscrypt_master_key *mk; 744 756 745 757 /* 746 758 * If ci is NULL, then the inode doesn't have an encryption key set up ··· 749 763 */ 750 764 if (!ci || !ci->ci_master_key) 751 765 return 0; 752 - mk = ci->ci_master_key->payload.data[0]; 753 766 754 767 /* 755 768 * With proper, non-racy use of FS_IOC_REMOVE_ENCRYPTION_KEY, all inodes ··· 767 782 * then the thread removing the key will either evict the inode itself 768 783 * or will correctly detect that it wasn't evicted due to the race. 769 784 */ 770 - return !is_master_key_secret_present(&mk->mk_secret); 785 + return !is_master_key_secret_present(&ci->ci_master_key->mk_secret); 771 786 } 772 787 EXPORT_SYMBOL_GPL(fscrypt_drop_inode);

+3 -1

fs/crypto/keysetup_v1.c

··· 143 143 144 144 /* Master key referenced by DIRECT_KEY policy */ 145 145 struct fscrypt_direct_key { 146 + struct super_block *dk_sb; 146 147 struct hlist_node dk_node; 147 148 refcount_t dk_refcount; 148 149 const struct fscrypt_mode *dk_mode; ··· 155 154 static void free_direct_key(struct fscrypt_direct_key *dk) 156 155 { 157 156 if (dk) { 158 - fscrypt_destroy_prepared_key(&dk->dk_key); 157 + fscrypt_destroy_prepared_key(dk->dk_sb, &dk->dk_key); 159 158 kfree_sensitive(dk); 160 159 } 161 160 } ··· 232 231 dk = kzalloc(sizeof(*dk), GFP_KERNEL); 233 232 if (!dk) 234 233 return ERR_PTR(-ENOMEM); 234 + dk->dk_sb = ci->ci_inode->i_sb; 235 235 refcount_set(&dk->dk_refcount, 1); 236 236 dk->dk_mode = ci->ci_mode; 237 237 err = fscrypt_prepare_key(&dk->dk_key, raw_key, ci);

+2 -19

fs/crypto/policy.c

··· 744 744 * delayed key setup that requires the inode number. 745 745 */ 746 746 if (ci->ci_policy.version == FSCRYPT_POLICY_V2 && 747 - (ci->ci_policy.v2.flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32)) { 748 - const struct fscrypt_master_key *mk = 749 - ci->ci_master_key->payload.data[0]; 750 - 751 - fscrypt_hash_inode_number(ci, mk); 752 - } 747 + (ci->ci_policy.v2.flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32)) 748 + fscrypt_hash_inode_number(ci, ci->ci_master_key); 753 749 754 750 return inode->i_sb->s_cop->set_context(inode, &ctx, ctxsize, fs_data); 755 751 } ··· 828 832 return fscrypt_policies_equal(p1->policy, p2->policy); 829 833 } 830 834 EXPORT_SYMBOL_GPL(fscrypt_dummy_policies_equal); 831 - 832 - /* Deprecated, do not use */ 833 - int fscrypt_set_test_dummy_encryption(struct super_block *sb, const char *arg, 834 - struct fscrypt_dummy_policy *dummy_policy) 835 - { 836 - struct fs_parameter param = { 837 - .type = fs_value_is_string, 838 - .string = arg ? (char *)arg : "", 839 - }; 840 - return fscrypt_parse_test_dummy_encryption(&param, dummy_policy) ?: 841 - fscrypt_add_test_dummy_key(sb, dummy_policy); 842 - } 843 - EXPORT_SYMBOL_GPL(fscrypt_set_test_dummy_encryption); 844 835 845 836 /** 846 837 * fscrypt_show_test_dummy_encryption() - show '-o test_dummy_encryption'

+6 -4

fs/ext4/readpage.c

··· 75 75 bio_for_each_segment_all(bv, bio, iter_all) { 76 76 page = bv->bv_page; 77 77 78 - /* PG_error was set if any post_read step failed */ 78 + /* PG_error was set if verity failed. */ 79 79 if (bio->bi_status || PageError(page)) { 80 80 ClearPageUptodate(page); 81 81 /* will re-read again later */ ··· 96 96 { 97 97 struct bio_post_read_ctx *ctx = 98 98 container_of(work, struct bio_post_read_ctx, work); 99 + struct bio *bio = ctx->bio; 99 100 100 - fscrypt_decrypt_bio(ctx->bio); 101 - 102 - bio_post_read_processing(ctx); 101 + if (fscrypt_decrypt_bio(bio)) 102 + bio_post_read_processing(ctx); 103 + else 104 + __read_end_io(bio); 103 105 } 104 106 105 107 static void verity_work(struct work_struct *work)

+10 -8

fs/f2fs/data.c

··· 139 139 continue; 140 140 } 141 141 142 - /* PG_error was set if decryption or verity failed. */ 142 + /* PG_error was set if verity failed. */ 143 143 if (bio->bi_status || PageError(page)) { 144 144 ClearPageUptodate(page); 145 145 /* will re-read again later */ ··· 185 185 struct page *page = bv->bv_page; 186 186 187 187 if (!f2fs_is_compressed_page(page) && 188 - !PageError(page) && !fsverity_verify_page(page)) 188 + !fsverity_verify_page(page)) 189 189 SetPageError(page); 190 190 } 191 191 } else { ··· 236 236 bio_for_each_segment_all(bv, ctx->bio, iter_all) { 237 237 struct page *page = bv->bv_page; 238 238 239 - /* PG_error was set if decryption failed. */ 240 239 if (f2fs_is_compressed_page(page)) 241 - f2fs_end_read_compressed_page(page, PageError(page), 242 - blkaddr, in_task); 240 + f2fs_end_read_compressed_page(page, false, blkaddr, 241 + in_task); 243 242 else 244 243 all_compressed = false; 245 244 ··· 258 259 { 259 260 struct bio_post_read_ctx *ctx = 260 261 container_of(work, struct bio_post_read_ctx, work); 262 + struct bio *bio = ctx->bio; 261 263 262 - if (ctx->enabled_steps & STEP_DECRYPT) 263 - fscrypt_decrypt_bio(ctx->bio); 264 + if ((ctx->enabled_steps & STEP_DECRYPT) && !fscrypt_decrypt_bio(bio)) { 265 + f2fs_finish_read_bio(bio, true); 266 + return; 267 + } 264 268 265 269 if (ctx->enabled_steps & STEP_DECOMPRESS) 266 270 f2fs_handle_step_decompress(ctx, true); 267 271 268 - f2fs_verify_and_finish_bio(ctx->bio, true); 272 + f2fs_verify_and_finish_bio(bio, true); 269 273 } 270 274 271 275 static void f2fs_read_end_io(struct bio *bio)

+13 -13

fs/f2fs/super.c

··· 3039 3039 *lblk_bits_ret = 8 * sizeof(block_t); 3040 3040 } 3041 3041 3042 - static int f2fs_get_num_devices(struct super_block *sb) 3042 + static struct block_device **f2fs_get_devices(struct super_block *sb, 3043 + unsigned int *num_devs) 3043 3044 { 3044 3045 struct f2fs_sb_info *sbi = F2FS_SB(sb); 3045 - 3046 - if (f2fs_is_multi_device(sbi)) 3047 - return sbi->s_ndevs; 3048 - return 1; 3049 - } 3050 - 3051 - static void f2fs_get_devices(struct super_block *sb, 3052 - struct request_queue **devs) 3053 - { 3054 - struct f2fs_sb_info *sbi = F2FS_SB(sb); 3046 + struct block_device **devs; 3055 3047 int i; 3056 3048 3049 + if (!f2fs_is_multi_device(sbi)) 3050 + return NULL; 3051 + 3052 + devs = kmalloc_array(sbi->s_ndevs, sizeof(*devs), GFP_KERNEL); 3053 + if (!devs) 3054 + return ERR_PTR(-ENOMEM); 3055 + 3057 3056 for (i = 0; i < sbi->s_ndevs; i++) 3058 - devs[i] = bdev_get_queue(FDEV(i).bdev); 3057 + devs[i] = FDEV(i).bdev; 3058 + *num_devs = sbi->s_ndevs; 3059 + return devs; 3059 3060 } 3060 3061 3061 3062 static const struct fscrypt_operations f2fs_cryptops = { ··· 3067 3066 .empty_dir = f2fs_empty_dir, 3068 3067 .has_stable_inodes = f2fs_has_stable_inodes, 3069 3068 .get_ino_and_lblk_bits = f2fs_get_ino_and_lblk_bits, 3070 - .get_num_devices = f2fs_get_num_devices, 3071 3069 .get_devices = f2fs_get_devices, 3072 3070 }; 3073 3071 #endif

+1 -1

fs/super.c

··· 291 291 WARN_ON(s->s_inode_lru.node); 292 292 WARN_ON(!list_empty(&s->s_mounts)); 293 293 security_sb_free(s); 294 - fscrypt_sb_free(s); 295 294 put_user_ns(s->s_user_ns); 296 295 kfree(s->s_subtype); 297 296 call_rcu(&s->rcu, destroy_super_rcu); ··· 479 480 evict_inodes(sb); 480 481 /* only nonzero refcount inodes can have marks */ 481 482 fsnotify_sb_delete(sb); 483 + fscrypt_sb_delete(sb); 482 484 security_sb_delete(sb); 483 485 484 486 if (sb->s_dio_done_wq) {

+1 -1

include/linux/fs.h

··· 1472 1472 const struct xattr_handler **s_xattr; 1473 1473 #ifdef CONFIG_FS_ENCRYPTION 1474 1474 const struct fscrypt_operations *s_cop; 1475 - struct key *s_master_keys; /* master crypto keys in use */ 1475 + struct fscrypt_keyring *s_master_keys; /* master crypto keys in use */ 1476 1476 #endif 1477 1477 #ifdef CONFIG_FS_VERITY 1478 1478 const struct fsverity_operations *s_vop;

+14 -18

include/linux/fscrypt.h

··· 161 161 int *ino_bits_ret, int *lblk_bits_ret); 162 162 163 163 /* 164 - * Return the number of block devices to which the filesystem may write 165 - * encrypted file contents. 164 + * Return an array of pointers to the block devices to which the 165 + * filesystem may write encrypted file contents, NULL if the filesystem 166 + * only has a single such block device, or an ERR_PTR() on error. 167 + * 168 + * On successful non-NULL return, *num_devs is set to the number of 169 + * devices in the returned array. The caller must free the returned 170 + * array using kfree(). 166 171 * 167 172 * If the filesystem can use multiple block devices (other than block 168 173 * devices that aren't used for encrypted file contents, such as 169 174 * external journal devices), and wants to support inline encryption, 170 175 * then it must implement this function. Otherwise it's not needed. 171 176 */ 172 - int (*get_num_devices)(struct super_block *sb); 173 - 174 - /* 175 - * If ->get_num_devices() returns a value greater than 1, then this 176 - * function is called to get the array of request_queues that the 177 - * filesystem is using -- one per block device. (There may be duplicate 178 - * entries in this array, as block devices can share a request_queue.) 179 - */ 180 - void (*get_devices)(struct super_block *sb, 181 - struct request_queue **devs); 177 + struct block_device **(*get_devices)(struct super_block *sb, 178 + unsigned int *num_devs); 182 179 }; 183 180 184 181 static inline struct fscrypt_info *fscrypt_get_info(const struct inode *inode) ··· 292 295 struct fscrypt_dummy_policy *dummy_policy); 293 296 bool fscrypt_dummy_policies_equal(const struct fscrypt_dummy_policy *p1, 294 297 const struct fscrypt_dummy_policy *p2); 295 - int fscrypt_set_test_dummy_encryption(struct super_block *sb, const char *arg, 296 - struct fscrypt_dummy_policy *dummy_policy); 297 298 void fscrypt_show_test_dummy_encryption(struct seq_file *seq, char sep, 298 299 struct super_block *sb); 299 300 static inline bool ··· 307 312 } 308 313 309 314 /* keyring.c */ 310 - void fscrypt_sb_free(struct super_block *sb); 315 + void fscrypt_sb_delete(struct super_block *sb); 311 316 int fscrypt_ioctl_add_key(struct file *filp, void __user *arg); 312 317 int fscrypt_add_test_dummy_key(struct super_block *sb, 313 318 const struct fscrypt_dummy_policy *dummy_policy); ··· 348 353 int fscrypt_d_revalidate(struct dentry *dentry, unsigned int flags); 349 354 350 355 /* bio.c */ 351 - void fscrypt_decrypt_bio(struct bio *bio); 356 + bool fscrypt_decrypt_bio(struct bio *bio); 352 357 int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk, 353 358 sector_t pblk, unsigned int len); 354 359 ··· 521 526 } 522 527 523 528 /* keyring.c */ 524 - static inline void fscrypt_sb_free(struct super_block *sb) 529 + static inline void fscrypt_sb_delete(struct super_block *sb) 525 530 { 526 531 } 527 532 ··· 641 646 } 642 647 643 648 /* bio.c */ 644 - static inline void fscrypt_decrypt_bio(struct bio *bio) 649 + static inline bool fscrypt_decrypt_bio(struct bio *bio) 645 650 { 651 + return true; 646 652 } 647 653 648 654 static inline int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk,

Configure Feed

Configure Feed