Merge tag 'dm-3.20-changes-2' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm

+14 -1

Documentation/device-mapper/dm-crypt.txt

··· 51 51 Otherwise #opt_params is the number of following arguments. 52 52 53 53 Example of optional parameters section: 54 - 1 allow_discards 54 + 3 allow_discards same_cpu_crypt submit_from_crypt_cpus 55 55 56 56 allow_discards 57 57 Block discard requests (a.k.a. TRIM) are passed through the crypt device. ··· 62 62 the leak of information about the ciphertext device (filesystem type, 63 63 used space etc.) if the discarded blocks can be located easily on the 64 64 device later. 65 + 66 + same_cpu_crypt 67 + Perform encryption using the same cpu that IO was submitted on. 68 + The default is to use an unbound workqueue so that encryption work 69 + is automatically balanced between available CPUs. 70 + 71 + submit_from_crypt_cpus 72 + Disable offloading writes to a separate thread after encryption. 73 + There are some situations where offloading write bios from the 74 + encryption threads to a single thread degrades performance 75 + significantly. The default is to offload write bios to the same 76 + thread because it benefits CFQ to have writes submitted using the 77 + same context. 65 78 66 79 Example scripts 67 80 ===============

+229 -171

drivers/md/dm-crypt.c

··· 18 18 #include <linux/slab.h> 19 19 #include <linux/crypto.h> 20 20 #include <linux/workqueue.h> 21 + #include <linux/kthread.h> 21 22 #include <linux/backing-dev.h> 22 23 #include <linux/atomic.h> 23 24 #include <linux/scatterlist.h> 25 + #include <linux/rbtree.h> 24 26 #include <asm/page.h> 25 27 #include <asm/unaligned.h> 26 28 #include <crypto/hash.h> ··· 60 58 atomic_t io_pending; 61 59 int error; 62 60 sector_t sector; 63 - struct dm_crypt_io *base_io; 61 + 62 + struct rb_node rb_node; 64 63 } CRYPTO_MINALIGN_ATTR; 65 64 66 65 struct dm_crypt_request { ··· 111 108 * Crypt: maps a linear range of a block device 112 109 * and encrypts / decrypts at the same time. 113 110 */ 114 - enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID }; 111 + enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID, 112 + DM_CRYPT_SAME_CPU, DM_CRYPT_NO_OFFLOAD }; 115 113 116 114 /* 117 115 * The fields in here must be read only after initialization. ··· 125 121 * pool for per bio private data, crypto requests and 126 122 * encryption requeusts/buffer pages 127 123 */ 128 - mempool_t *io_pool; 129 124 mempool_t *req_pool; 130 125 mempool_t *page_pool; 131 126 struct bio_set *bs; 127 + struct mutex bio_alloc_lock; 132 128 133 129 struct workqueue_struct *io_queue; 134 130 struct workqueue_struct *crypt_queue; 131 + 132 + struct task_struct *write_thread; 133 + wait_queue_head_t write_thread_wait; 134 + struct rb_root write_tree; 135 135 136 136 char *cipher; 137 137 char *cipher_string; ··· 180 172 }; 181 173 182 174 #define MIN_IOS 16 183 - #define MIN_POOL_PAGES 32 184 - 185 - static struct kmem_cache *_crypt_io_pool; 186 175 187 176 static void clone_init(struct dm_crypt_io *, struct bio *); 188 177 static void kcryptd_queue_crypt(struct dm_crypt_io *io); ··· 951 946 return 0; 952 947 } 953 948 949 + static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone); 950 + 954 951 /* 955 952 * Generate a new unfragmented bio with the given size 956 953 * This should never violate the device limitations 957 - * May return a smaller bio when running out of pages, indicated by 958 - * *out_of_pages set to 1. 954 + * 955 + * This function may be called concurrently. If we allocate from the mempool 956 + * concurrently, there is a possibility of deadlock. For example, if we have 957 + * mempool of 256 pages, two processes, each wanting 256, pages allocate from 958 + * the mempool concurrently, it may deadlock in a situation where both processes 959 + * have allocated 128 pages and the mempool is exhausted. 960 + * 961 + * In order to avoid this scenario we allocate the pages under a mutex. 962 + * 963 + * In order to not degrade performance with excessive locking, we try 964 + * non-blocking allocations without a mutex first but on failure we fallback 965 + * to blocking allocations with a mutex. 959 966 */ 960 - static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size, 961 - unsigned *out_of_pages) 967 + static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size) 962 968 { 963 969 struct crypt_config *cc = io->cc; 964 970 struct bio *clone; 965 971 unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 966 - gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM; 967 - unsigned i, len; 972 + gfp_t gfp_mask = GFP_NOWAIT | __GFP_HIGHMEM; 973 + unsigned i, len, remaining_size; 968 974 struct page *page; 975 + struct bio_vec *bvec; 976 + 977 + retry: 978 + if (unlikely(gfp_mask & __GFP_WAIT)) 979 + mutex_lock(&cc->bio_alloc_lock); 969 980 970 981 clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs); 971 982 if (!clone) 972 - return NULL; 983 + goto return_clone; 973 984 974 985 clone_init(io, clone); 975 - *out_of_pages = 0; 986 + 987 + remaining_size = size; 976 988 977 989 for (i = 0; i < nr_iovecs; i++) { 978 990 page = mempool_alloc(cc->page_pool, gfp_mask); 979 991 if (!page) { 980 - *out_of_pages = 1; 981 - break; 992 + crypt_free_buffer_pages(cc, clone); 993 + bio_put(clone); 994 + gfp_mask |= __GFP_WAIT; 995 + goto retry; 982 996 } 983 997 984 - /* 985 - * If additional pages cannot be allocated without waiting, 986 - * return a partially-allocated bio. The caller will then try 987 - * to allocate more bios while submitting this partial bio. 988 - */ 989 - gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT; 998 + len = (remaining_size > PAGE_SIZE) ? PAGE_SIZE : remaining_size; 990 999 991 - len = (size > PAGE_SIZE) ? PAGE_SIZE : size; 1000 + bvec = &clone->bi_io_vec[clone->bi_vcnt++]; 1001 + bvec->bv_page = page; 1002 + bvec->bv_len = len; 1003 + bvec->bv_offset = 0; 992 1004 993 - if (!bio_add_page(clone, page, len, 0)) { 994 - mempool_free(page, cc->page_pool); 995 - break; 996 - } 1005 + clone->bi_iter.bi_size += len; 997 1006 998 - size -= len; 1007 + remaining_size -= len; 999 1008 } 1000 1009 1001 - if (!clone->bi_iter.bi_size) { 1002 - bio_put(clone); 1003 - return NULL; 1004 - } 1010 + return_clone: 1011 + if (unlikely(gfp_mask & __GFP_WAIT)) 1012 + mutex_unlock(&cc->bio_alloc_lock); 1005 1013 1006 1014 return clone; 1007 1015 } ··· 1038 1020 io->base_bio = bio; 1039 1021 io->sector = sector; 1040 1022 io->error = 0; 1041 - io->base_io = NULL; 1042 1023 io->ctx.req = NULL; 1043 1024 atomic_set(&io->io_pending, 0); 1044 1025 } ··· 1050 1033 /* 1051 1034 * One of the bios was finished. Check for completion of 1052 1035 * the whole request and correctly clean up the buffer. 1053 - * If base_io is set, wait for the last fragment to complete. 1054 1036 */ 1055 1037 static void crypt_dec_pending(struct dm_crypt_io *io) 1056 1038 { 1057 1039 struct crypt_config *cc = io->cc; 1058 1040 struct bio *base_bio = io->base_bio; 1059 - struct dm_crypt_io *base_io = io->base_io; 1060 1041 int error = io->error; 1061 1042 1062 1043 if (!atomic_dec_and_test(&io->io_pending)) ··· 1062 1047 1063 1048 if (io->ctx.req) 1064 1049 crypt_free_req(cc, io->ctx.req, base_bio); 1065 - if (io != dm_per_bio_data(base_bio, cc->per_bio_data_size)) 1066 - mempool_free(io, cc->io_pool); 1067 1050 1068 - if (likely(!base_io)) 1069 - bio_endio(base_bio, error); 1070 - else { 1071 - if (error && !base_io->error) 1072 - base_io->error = error; 1073 - crypt_dec_pending(base_io); 1074 - } 1051 + bio_endio(base_bio, error); 1075 1052 } 1076 1053 1077 1054 /* ··· 1145 1138 return 0; 1146 1139 } 1147 1140 1148 - static void kcryptd_io_write(struct dm_crypt_io *io) 1149 - { 1150 - struct bio *clone = io->ctx.bio_out; 1151 - generic_make_request(clone); 1152 - } 1153 - 1154 - static void kcryptd_io(struct work_struct *work) 1141 + static void kcryptd_io_read_work(struct work_struct *work) 1155 1142 { 1156 1143 struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work); 1157 1144 1158 - if (bio_data_dir(io->base_bio) == READ) { 1159 - crypt_inc_pending(io); 1160 - if (kcryptd_io_read(io, GFP_NOIO)) 1161 - io->error = -ENOMEM; 1162 - crypt_dec_pending(io); 1163 - } else 1164 - kcryptd_io_write(io); 1145 + crypt_inc_pending(io); 1146 + if (kcryptd_io_read(io, GFP_NOIO)) 1147 + io->error = -ENOMEM; 1148 + crypt_dec_pending(io); 1165 1149 } 1166 1150 1167 - static void kcryptd_queue_io(struct dm_crypt_io *io) 1151 + static void kcryptd_queue_read(struct dm_crypt_io *io) 1168 1152 { 1169 1153 struct crypt_config *cc = io->cc; 1170 1154 1171 - INIT_WORK(&io->work, kcryptd_io); 1155 + INIT_WORK(&io->work, kcryptd_io_read_work); 1172 1156 queue_work(cc->io_queue, &io->work); 1157 + } 1158 + 1159 + static void kcryptd_io_write(struct dm_crypt_io *io) 1160 + { 1161 + struct bio *clone = io->ctx.bio_out; 1162 + 1163 + generic_make_request(clone); 1164 + } 1165 + 1166 + #define crypt_io_from_node(node) rb_entry((node), struct dm_crypt_io, rb_node) 1167 + 1168 + static int dmcrypt_write(void *data) 1169 + { 1170 + struct crypt_config *cc = data; 1171 + struct dm_crypt_io *io; 1172 + 1173 + while (1) { 1174 + struct rb_root write_tree; 1175 + struct blk_plug plug; 1176 + 1177 + DECLARE_WAITQUEUE(wait, current); 1178 + 1179 + spin_lock_irq(&cc->write_thread_wait.lock); 1180 + continue_locked: 1181 + 1182 + if (!RB_EMPTY_ROOT(&cc->write_tree)) 1183 + goto pop_from_list; 1184 + 1185 + __set_current_state(TASK_INTERRUPTIBLE); 1186 + __add_wait_queue(&cc->write_thread_wait, &wait); 1187 + 1188 + spin_unlock_irq(&cc->write_thread_wait.lock); 1189 + 1190 + if (unlikely(kthread_should_stop())) { 1191 + set_task_state(current, TASK_RUNNING); 1192 + remove_wait_queue(&cc->write_thread_wait, &wait); 1193 + break; 1194 + } 1195 + 1196 + schedule(); 1197 + 1198 + set_task_state(current, TASK_RUNNING); 1199 + spin_lock_irq(&cc->write_thread_wait.lock); 1200 + __remove_wait_queue(&cc->write_thread_wait, &wait); 1201 + goto continue_locked; 1202 + 1203 + pop_from_list: 1204 + write_tree = cc->write_tree; 1205 + cc->write_tree = RB_ROOT; 1206 + spin_unlock_irq(&cc->write_thread_wait.lock); 1207 + 1208 + BUG_ON(rb_parent(write_tree.rb_node)); 1209 + 1210 + /* 1211 + * Note: we cannot walk the tree here with rb_next because 1212 + * the structures may be freed when kcryptd_io_write is called. 1213 + */ 1214 + blk_start_plug(&plug); 1215 + do { 1216 + io = crypt_io_from_node(rb_first(&write_tree)); 1217 + rb_erase(&io->rb_node, &write_tree); 1218 + kcryptd_io_write(io); 1219 + } while (!RB_EMPTY_ROOT(&write_tree)); 1220 + blk_finish_plug(&plug); 1221 + } 1222 + return 0; 1173 1223 } 1174 1224 1175 1225 static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async) 1176 1226 { 1177 1227 struct bio *clone = io->ctx.bio_out; 1178 1228 struct crypt_config *cc = io->cc; 1229 + unsigned long flags; 1230 + sector_t sector; 1231 + struct rb_node **rbp, *parent; 1179 1232 1180 1233 if (unlikely(io->error < 0)) { 1181 1234 crypt_free_buffer_pages(cc, clone); ··· 1249 1182 1250 1183 clone->bi_iter.bi_sector = cc->start + io->sector; 1251 1184 1252 - if (async) 1253 - kcryptd_queue_io(io); 1254 - else 1185 + if (likely(!async) && test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) { 1255 1186 generic_make_request(clone); 1187 + return; 1188 + } 1189 + 1190 + spin_lock_irqsave(&cc->write_thread_wait.lock, flags); 1191 + rbp = &cc->write_tree.rb_node; 1192 + parent = NULL; 1193 + sector = io->sector; 1194 + while (*rbp) { 1195 + parent = *rbp; 1196 + if (sector < crypt_io_from_node(parent)->sector) 1197 + rbp = &(*rbp)->rb_left; 1198 + else 1199 + rbp = &(*rbp)->rb_right; 1200 + } 1201 + rb_link_node(&io->rb_node, parent, rbp); 1202 + rb_insert_color(&io->rb_node, &cc->write_tree); 1203 + 1204 + wake_up_locked(&cc->write_thread_wait); 1205 + spin_unlock_irqrestore(&cc->write_thread_wait.lock, flags); 1256 1206 } 1257 1207 1258 1208 static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) 1259 1209 { 1260 1210 struct crypt_config *cc = io->cc; 1261 1211 struct bio *clone; 1262 - struct dm_crypt_io *new_io; 1263 1212 int crypt_finished; 1264 - unsigned out_of_pages = 0; 1265 - unsigned remaining = io->base_bio->bi_iter.bi_size; 1266 1213 sector_t sector = io->sector; 1267 1214 int r; 1268 1215 ··· 1286 1205 crypt_inc_pending(io); 1287 1206 crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, sector); 1288 1207 1289 - /* 1290 - * The allocated buffers can be smaller than the whole bio, 1291 - * so repeat the whole process until all the data can be handled. 1292 - */ 1293 - while (remaining) { 1294 - clone = crypt_alloc_buffer(io, remaining, &out_of_pages); 1295 - if (unlikely(!clone)) { 1296 - io->error = -ENOMEM; 1297 - break; 1298 - } 1299 - 1300 - io->ctx.bio_out = clone; 1301 - io->ctx.iter_out = clone->bi_iter; 1302 - 1303 - remaining -= clone->bi_iter.bi_size; 1304 - sector += bio_sectors(clone); 1305 - 1306 - crypt_inc_pending(io); 1307 - 1308 - r = crypt_convert(cc, &io->ctx); 1309 - if (r < 0) 1310 - io->error = -EIO; 1311 - 1312 - crypt_finished = atomic_dec_and_test(&io->ctx.cc_pending); 1313 - 1314 - /* Encryption was already finished, submit io now */ 1315 - if (crypt_finished) { 1316 - kcryptd_crypt_write_io_submit(io, 0); 1317 - 1318 - /* 1319 - * If there was an error, do not try next fragments. 1320 - * For async, error is processed in async handler. 1321 - */ 1322 - if (unlikely(r < 0)) 1323 - break; 1324 - 1325 - io->sector = sector; 1326 - } 1327 - 1328 - /* 1329 - * Out of memory -> run queues 1330 - * But don't wait if split was due to the io size restriction 1331 - */ 1332 - if (unlikely(out_of_pages)) 1333 - congestion_wait(BLK_RW_ASYNC, HZ/100); 1334 - 1335 - /* 1336 - * With async crypto it is unsafe to share the crypto context 1337 - * between fragments, so switch to a new dm_crypt_io structure. 1338 - */ 1339 - if (unlikely(!crypt_finished && remaining)) { 1340 - new_io = mempool_alloc(cc->io_pool, GFP_NOIO); 1341 - crypt_io_init(new_io, io->cc, io->base_bio, sector); 1342 - crypt_inc_pending(new_io); 1343 - crypt_convert_init(cc, &new_io->ctx, NULL, 1344 - io->base_bio, sector); 1345 - new_io->ctx.iter_in = io->ctx.iter_in; 1346 - 1347 - /* 1348 - * Fragments after the first use the base_io 1349 - * pending count. 1350 - */ 1351 - if (!io->base_io) 1352 - new_io->base_io = io; 1353 - else { 1354 - new_io->base_io = io->base_io; 1355 - crypt_inc_pending(io->base_io); 1356 - crypt_dec_pending(io); 1357 - } 1358 - 1359 - io = new_io; 1360 - } 1208 + clone = crypt_alloc_buffer(io, io->base_bio->bi_iter.bi_size); 1209 + if (unlikely(!clone)) { 1210 + io->error = -EIO; 1211 + goto dec; 1361 1212 } 1362 1213 1214 + io->ctx.bio_out = clone; 1215 + io->ctx.iter_out = clone->bi_iter; 1216 + 1217 + sector += bio_sectors(clone); 1218 + 1219 + crypt_inc_pending(io); 1220 + r = crypt_convert(cc, &io->ctx); 1221 + if (r) 1222 + io->error = -EIO; 1223 + crypt_finished = atomic_dec_and_test(&io->ctx.cc_pending); 1224 + 1225 + /* Encryption was already finished, submit io now */ 1226 + if (crypt_finished) { 1227 + kcryptd_crypt_write_io_submit(io, 0); 1228 + io->sector = sector; 1229 + } 1230 + 1231 + dec: 1363 1232 crypt_dec_pending(io); 1364 1233 } 1365 1234 ··· 1512 1481 if (!cc) 1513 1482 return; 1514 1483 1484 + if (cc->write_thread) 1485 + kthread_stop(cc->write_thread); 1486 + 1515 1487 if (cc->io_queue) 1516 1488 destroy_workqueue(cc->io_queue); 1517 1489 if (cc->crypt_queue) ··· 1529 1495 mempool_destroy(cc->page_pool); 1530 1496 if (cc->req_pool) 1531 1497 mempool_destroy(cc->req_pool); 1532 - if (cc->io_pool) 1533 - mempool_destroy(cc->io_pool); 1534 1498 1535 1499 if (cc->iv_gen_ops && cc->iv_gen_ops->dtr) 1536 1500 cc->iv_gen_ops->dtr(cc); ··· 1720 1688 char dummy; 1721 1689 1722 1690 static struct dm_arg _args[] = { 1723 - {0, 1, "Invalid number of feature args"}, 1691 + {0, 3, "Invalid number of feature args"}, 1724 1692 }; 1725 1693 1726 1694 if (argc < 5) { ··· 1742 1710 if (ret < 0) 1743 1711 goto bad; 1744 1712 1745 - ret = -ENOMEM; 1746 - cc->io_pool = mempool_create_slab_pool(MIN_IOS, _crypt_io_pool); 1747 - if (!cc->io_pool) { 1748 - ti->error = "Cannot allocate crypt io mempool"; 1749 - goto bad; 1750 - } 1751 - 1752 1713 cc->dmreq_start = sizeof(struct ablkcipher_request); 1753 1714 cc->dmreq_start += crypto_ablkcipher_reqsize(any_tfm(cc)); 1754 1715 cc->dmreq_start = ALIGN(cc->dmreq_start, __alignof__(struct dm_crypt_request)); ··· 1759 1734 iv_size_padding = crypto_ablkcipher_alignmask(any_tfm(cc)); 1760 1735 } 1761 1736 1737 + ret = -ENOMEM; 1762 1738 cc->req_pool = mempool_create_kmalloc_pool(MIN_IOS, cc->dmreq_start + 1763 1739 sizeof(struct dm_crypt_request) + iv_size_padding + cc->iv_size); 1764 1740 if (!cc->req_pool) { ··· 1772 1746 sizeof(struct dm_crypt_request) + iv_size_padding + cc->iv_size, 1773 1747 ARCH_KMALLOC_MINALIGN); 1774 1748 1775 - cc->page_pool = mempool_create_page_pool(MIN_POOL_PAGES, 0); 1749 + cc->page_pool = mempool_create_page_pool(BIO_MAX_PAGES, 0); 1776 1750 if (!cc->page_pool) { 1777 1751 ti->error = "Cannot allocate page mempool"; 1778 1752 goto bad; ··· 1783 1757 ti->error = "Cannot allocate crypt bioset"; 1784 1758 goto bad; 1785 1759 } 1760 + 1761 + mutex_init(&cc->bio_alloc_lock); 1786 1762 1787 1763 ret = -EINVAL; 1788 1764 if (sscanf(argv[2], "%llu%c", &tmpll, &dummy) != 1) { ··· 1816 1788 if (ret) 1817 1789 goto bad; 1818 1790 1819 - opt_string = dm_shift_arg(&as); 1791 + while (opt_params--) { 1792 + opt_string = dm_shift_arg(&as); 1793 + if (!opt_string) { 1794 + ti->error = "Not enough feature arguments"; 1795 + goto bad; 1796 + } 1820 1797 1821 - if (opt_params == 1 && opt_string && 1822 - !strcasecmp(opt_string, "allow_discards")) 1823 - ti->num_discard_bios = 1; 1824 - else if (opt_params) { 1825 - ret = -EINVAL; 1826 - ti->error = "Invalid feature arguments"; 1827 - goto bad; 1798 + if (!strcasecmp(opt_string, "allow_discards")) 1799 + ti->num_discard_bios = 1; 1800 + 1801 + else if (!strcasecmp(opt_string, "same_cpu_crypt")) 1802 + set_bit(DM_CRYPT_SAME_CPU, &cc->flags); 1803 + 1804 + else if (!strcasecmp(opt_string, "submit_from_crypt_cpus")) 1805 + set_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags); 1806 + 1807 + else { 1808 + ti->error = "Invalid feature arguments"; 1809 + goto bad; 1810 + } 1828 1811 } 1829 1812 } 1830 1813 ··· 1846 1807 goto bad; 1847 1808 } 1848 1809 1849 - cc->crypt_queue = alloc_workqueue("kcryptd", 1850 - WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 1); 1810 + if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags)) 1811 + cc->crypt_queue = alloc_workqueue("kcryptd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 1); 1812 + else 1813 + cc->crypt_queue = alloc_workqueue("kcryptd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, 1814 + num_online_cpus()); 1851 1815 if (!cc->crypt_queue) { 1852 1816 ti->error = "Couldn't create kcryptd queue"; 1853 1817 goto bad; 1854 1818 } 1819 + 1820 + init_waitqueue_head(&cc->write_thread_wait); 1821 + cc->write_tree = RB_ROOT; 1822 + 1823 + cc->write_thread = kthread_create(dmcrypt_write, cc, "dmcrypt_write"); 1824 + if (IS_ERR(cc->write_thread)) { 1825 + ret = PTR_ERR(cc->write_thread); 1826 + cc->write_thread = NULL; 1827 + ti->error = "Couldn't spawn write thread"; 1828 + goto bad; 1829 + } 1830 + wake_up_process(cc->write_thread); 1855 1831 1856 1832 ti->num_flush_bios = 1; 1857 1833 ti->discard_zeroes_data_unsupported = true; ··· 1902 1848 1903 1849 if (bio_data_dir(io->base_bio) == READ) { 1904 1850 if (kcryptd_io_read(io, GFP_NOWAIT)) 1905 - kcryptd_queue_io(io); 1851 + kcryptd_queue_read(io); 1906 1852 } else 1907 1853 kcryptd_queue_crypt(io); 1908 1854 ··· 1914 1860 { 1915 1861 struct crypt_config *cc = ti->private; 1916 1862 unsigned i, sz = 0; 1863 + int num_feature_args = 0; 1917 1864 1918 1865 switch (type) { 1919 1866 case STATUSTYPE_INFO: ··· 1933 1878 DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset, 1934 1879 cc->dev->name, (unsigned long long)cc->start); 1935 1880 1936 - if (ti->num_discard_bios) 1937 - DMEMIT(" 1 allow_discards"); 1881 + num_feature_args += !!ti->num_discard_bios; 1882 + num_feature_args += test_bit(DM_CRYPT_SAME_CPU, &cc->flags); 1883 + num_feature_args += test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags); 1884 + if (num_feature_args) { 1885 + DMEMIT(" %d", num_feature_args); 1886 + if (ti->num_discard_bios) 1887 + DMEMIT(" allow_discards"); 1888 + if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags)) 1889 + DMEMIT(" same_cpu_crypt"); 1890 + if (test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) 1891 + DMEMIT(" submit_from_crypt_cpus"); 1892 + } 1938 1893 1939 1894 break; 1940 1895 } ··· 2041 1976 2042 1977 static struct target_type crypt_target = { 2043 1978 .name = "crypt", 2044 - .version = {1, 13, 0}, 1979 + .version = {1, 14, 0}, 2045 1980 .module = THIS_MODULE, 2046 1981 .ctr = crypt_ctr, 2047 1982 .dtr = crypt_dtr, ··· 2059 1994 { 2060 1995 int r; 2061 1996 2062 - _crypt_io_pool = KMEM_CACHE(dm_crypt_io, 0); 2063 - if (!_crypt_io_pool) 2064 - return -ENOMEM; 2065 - 2066 1997 r = dm_register_target(&crypt_target); 2067 - if (r < 0) { 1998 + if (r < 0) 2068 1999 DMERR("register failed %d", r); 2069 - kmem_cache_destroy(_crypt_io_pool); 2070 - } 2071 2000 2072 2001 return r; 2073 2002 } ··· 2069 2010 static void __exit dm_crypt_exit(void) 2070 2011 { 2071 2012 dm_unregister_target(&crypt_target); 2072 - kmem_cache_destroy(_crypt_io_pool); 2073 2013 } 2074 2014 2075 2015 module_init(dm_crypt_init);

+6

drivers/md/dm-io.c

··· 290 290 unsigned short logical_block_size = queue_logical_block_size(q); 291 291 sector_t num_sectors; 292 292 293 + /* Reject unsupported discard requests */ 294 + if ((rw & REQ_DISCARD) && !blk_queue_discard(q)) { 295 + dec_count(io, region, -EOPNOTSUPP); 296 + return; 297 + } 298 + 293 299 /* 294 300 * where->count may be zero if rw holds a flush and we need to 295 301 * send a zero-sized flush.

+9

drivers/md/dm-raid1.c

··· 604 604 return; 605 605 } 606 606 607 + /* 608 + * If the bio is discard, return an error, but do not 609 + * degrade the array. 610 + */ 611 + if (bio->bi_rw & REQ_DISCARD) { 612 + bio_endio(bio, -EOPNOTSUPP); 613 + return; 614 + } 615 + 607 616 for (i = 0; i < ms->nr_mirrors; i++) 608 617 if (test_bit(i, &error)) 609 618 fail_mirror(ms->mirror + i, DM_RAID1_WRITE_ERROR);

+2 -2

drivers/md/dm-snap.c

··· 1432 1432 full_bio->bi_private = pe->full_bio_private; 1433 1433 atomic_inc(&full_bio->bi_remaining); 1434 1434 } 1435 - free_pending_exception(pe); 1436 - 1437 1435 increment_pending_exceptions_done_count(); 1438 1436 1439 1437 up_write(&s->lock); ··· 1448 1450 } 1449 1451 1450 1452 retry_origin_bios(s, origin_bios); 1453 + 1454 + free_pending_exception(pe); 1451 1455 } 1452 1456 1453 1457 static void commit_callback(void *context, int success)

+10 -17

drivers/md/dm.c

··· 2571 2571 return 0; 2572 2572 } 2573 2573 2574 - static struct mapped_device *dm_find_md(dev_t dev) 2574 + struct mapped_device *dm_get_md(dev_t dev) 2575 2575 { 2576 2576 struct mapped_device *md; 2577 2577 unsigned minor = MINOR(dev); ··· 2582 2582 spin_lock(&_minor_lock); 2583 2583 2584 2584 md = idr_find(&_minor_idr, minor); 2585 - if (md && (md == MINOR_ALLOCED || 2586 - (MINOR(disk_devt(dm_disk(md))) != minor) || 2587 - dm_deleting_md(md) || 2588 - test_bit(DMF_FREEING, &md->flags))) { 2589 - md = NULL; 2590 - goto out; 2585 + if (md) { 2586 + if ((md == MINOR_ALLOCED || 2587 + (MINOR(disk_devt(dm_disk(md))) != minor) || 2588 + dm_deleting_md(md) || 2589 + test_bit(DMF_FREEING, &md->flags))) { 2590 + md = NULL; 2591 + goto out; 2592 + } 2593 + dm_get(md); 2591 2594 } 2592 2595 2593 2596 out: 2594 2597 spin_unlock(&_minor_lock); 2595 - 2596 - return md; 2597 - } 2598 - 2599 - struct mapped_device *dm_get_md(dev_t dev) 2600 - { 2601 - struct mapped_device *md = dm_find_md(dev); 2602 - 2603 - if (md) 2604 - dm_get(md); 2605 2598 2606 2599 return md; 2607 2600 }

+3 -1

drivers/md/persistent-data/dm-space-map-disk.c

··· 78 78 if (r) 79 79 return r; 80 80 81 - return count > 1; 81 + *result = count > 1; 82 + 83 + return 0; 82 84 } 83 85 84 86 static int sm_disk_set_count(struct dm_space_map *sm, dm_block_t b,

Configure Feed

Configure Feed