Merge branch 'stable/for-linus-4.10' of git://git.kernel.org/pub/scm/linux/kernel/git/konrad/swiotlb

+2 -1

Documentation/admin-guide/kernel-parameters.txt

··· 3821 3821 it if 0 is given (See Documentation/cgroup-v1/memory.txt) 3822 3822 3823 3823 swiotlb= [ARM,IA-64,PPC,MIPS,X86] 3824 - Format: { <int> | force } 3824 + Format: { <int> | force | noforce } 3825 3825 <int> -- Number of I/O TLB slabs 3826 3826 force -- force using of bounce buffers even if they 3827 3827 wouldn't be automatically used by the kernel 3828 + noforce -- Never use bounce buffers (for debugging) 3828 3829 3829 3830 switches= [HW,M68k] 3830 3831

+2 -1

arch/arm64/mm/dma-mapping.c

··· 524 524 525 525 static int __init arm64_dma_init(void) 526 526 { 527 - if (swiotlb_force || max_pfn > (arm64_dma_phys_limit >> PAGE_SHIFT)) 527 + if (swiotlb_force == SWIOTLB_FORCE || 528 + max_pfn > (arm64_dma_phys_limit >> PAGE_SHIFT)) 528 529 swiotlb = 1; 529 530 530 531 return atomic_pool_init();

+2 -1

arch/arm64/mm/init.c

··· 401 401 */ 402 402 void __init mem_init(void) 403 403 { 404 - if (swiotlb_force || max_pfn > (arm64_dma_phys_limit >> PAGE_SHIFT)) 404 + if (swiotlb_force == SWIOTLB_FORCE || 405 + max_pfn > (arm64_dma_phys_limit >> PAGE_SHIFT)) 405 406 swiotlb_init(1); 406 407 407 408 set_max_mapnr(pfn_to_page(max_pfn) - mem_map);

+2 -4

arch/x86/kernel/pci-swiotlb.c

··· 68 68 */ 69 69 int __init pci_swiotlb_detect_override(void) 70 70 { 71 - int use_swiotlb = swiotlb | swiotlb_force; 72 - 73 - if (swiotlb_force) 71 + if (swiotlb_force == SWIOTLB_FORCE) 74 72 swiotlb = 1; 75 73 76 - return use_swiotlb; 74 + return swiotlb; 77 75 } 78 76 IOMMU_INIT_FINISH(pci_swiotlb_detect_override, 79 77 pci_xen_swiotlb_detect,

+1 -1

arch/x86/xen/pci-swiotlb-xen.c

··· 48 48 * activate this IOMMU. If running as PV privileged, activate it 49 49 * irregardless. 50 50 */ 51 - if ((xen_initial_domain() || swiotlb || swiotlb_force)) 51 + if (xen_initial_domain() || swiotlb || swiotlb_force == SWIOTLB_FORCE) 52 52 xen_swiotlb = 1; 53 53 54 54 /* If we are running under Xen, we MUST disable the native SWIOTLB.

+1 -10

drivers/gpu/drm/i915/i915_gem.c

··· 2306 2306 mutex_unlock(&obj->mm.lock); 2307 2307 } 2308 2308 2309 - static unsigned int swiotlb_max_size(void) 2310 - { 2311 - #if IS_ENABLED(CONFIG_SWIOTLB) 2312 - return rounddown(swiotlb_nr_tbl() << IO_TLB_SHIFT, PAGE_SIZE); 2313 - #else 2314 - return 0; 2315 - #endif 2316 - } 2317 - 2318 2309 static void i915_sg_trim(struct sg_table *orig_st) 2319 2310 { 2320 2311 struct sg_table new_st; ··· 2353 2362 GEM_BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS); 2354 2363 GEM_BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS); 2355 2364 2356 - max_segment = swiotlb_max_size(); 2365 + max_segment = swiotlb_max_segment(); 2357 2366 if (!max_segment) 2358 2367 max_segment = rounddown(UINT_MAX, PAGE_SIZE); 2359 2368

+6 -2

drivers/xen/swiotlb-xen.c

··· 275 275 rc = 0; 276 276 } else 277 277 rc = swiotlb_late_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs); 278 + 279 + if (!rc) 280 + swiotlb_set_max_segment(PAGE_SIZE); 281 + 278 282 return rc; 279 283 error: 280 284 if (repeat--) { ··· 396 392 if (dma_capable(dev, dev_addr, size) && 397 393 !range_straddles_page_boundary(phys, size) && 398 394 !xen_arch_need_swiotlb(dev, phys, dev_addr) && 399 - !swiotlb_force) { 395 + (swiotlb_force != SWIOTLB_FORCE)) { 400 396 /* we are not interested in the dma_addr returned by 401 397 * xen_dma_map_page, only in the potential cache flushes executed 402 398 * by the function. */ ··· 556 552 phys_addr_t paddr = sg_phys(sg); 557 553 dma_addr_t dev_addr = xen_phys_to_bus(paddr); 558 554 559 - if (swiotlb_force || 555 + if (swiotlb_force == SWIOTLB_FORCE || 560 556 xen_arch_need_swiotlb(hwdev, paddr, dev_addr) || 561 557 !dma_capable(hwdev, dev_addr, sg->length) || 562 558 range_straddles_page_boundary(paddr, sg->length)) {

+10 -1

include/linux/swiotlb.h

··· 9 9 struct page; 10 10 struct scatterlist; 11 11 12 - extern int swiotlb_force; 12 + enum swiotlb_force { 13 + SWIOTLB_NORMAL, /* Default - depending on HW DMA mask etc. */ 14 + SWIOTLB_FORCE, /* swiotlb=force */ 15 + SWIOTLB_NO_FORCE, /* swiotlb=noforce */ 16 + }; 17 + 18 + extern enum swiotlb_force swiotlb_force; 13 19 14 20 /* 15 21 * Maximum allowable number of contiguous slabs to map, ··· 114 108 115 109 #ifdef CONFIG_SWIOTLB 116 110 extern void __init swiotlb_free(void); 111 + unsigned int swiotlb_max_segment(void); 117 112 #else 118 113 static inline void swiotlb_free(void) { } 114 + static inline unsigned int swiotlb_max_segment(void) { return 0; } 119 115 #endif 120 116 121 117 extern void swiotlb_print_info(void); 122 118 extern int is_swiotlb_buffer(phys_addr_t paddr); 119 + extern void swiotlb_set_max_segment(unsigned int); 123 120 124 121 #endif /* __LINUX_SWIOTLB_H */

+10 -7

include/trace/events/swiotlb.h

··· 11 11 TP_PROTO(struct device *dev, 12 12 dma_addr_t dev_addr, 13 13 size_t size, 14 - int swiotlb_force), 14 + enum swiotlb_force swiotlb_force), 15 15 16 16 TP_ARGS(dev, dev_addr, size, swiotlb_force), 17 17 18 18 TP_STRUCT__entry( 19 - __string( dev_name, dev_name(dev) ) 20 - __field( u64, dma_mask ) 21 - __field( dma_addr_t, dev_addr ) 22 - __field( size_t, size ) 23 - __field( int, swiotlb_force ) 19 + __string( dev_name, dev_name(dev) ) 20 + __field( u64, dma_mask ) 21 + __field( dma_addr_t, dev_addr ) 22 + __field( size_t, size ) 23 + __field( enum swiotlb_force, swiotlb_force ) 24 24 ), 25 25 26 26 TP_fast_assign( ··· 37 37 __entry->dma_mask, 38 38 (unsigned long long)__entry->dev_addr, 39 39 __entry->size, 40 - __entry->swiotlb_force ? "swiotlb_force" : "" ) 40 + __print_symbolic(__entry->swiotlb_force, 41 + { SWIOTLB_NORMAL, "NORMAL" }, 42 + { SWIOTLB_FORCE, "FORCE" }, 43 + { SWIOTLB_NO_FORCE, "NO_FORCE" })) 41 44 ); 42 45 43 46 #endif /* _TRACE_SWIOTLB_H */

+46 -6

lib/swiotlb.c

··· 53 53 */ 54 54 #define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT) 55 55 56 - int swiotlb_force; 56 + enum swiotlb_force swiotlb_force; 57 57 58 58 /* 59 59 * Used to do a quick range check in swiotlb_tbl_unmap_single and ··· 83 83 static unsigned int io_tlb_index; 84 84 85 85 /* 86 + * Max segment that we can provide which (if pages are contingous) will 87 + * not be bounced (unless SWIOTLB_FORCE is set). 88 + */ 89 + unsigned int max_segment; 90 + 91 + /* 86 92 * We need to save away the original address corresponding to a mapped entry 87 93 * for the sync operations. 88 94 */ ··· 112 106 } 113 107 if (*str == ',') 114 108 ++str; 115 - if (!strcmp(str, "force")) 116 - swiotlb_force = 1; 109 + if (!strcmp(str, "force")) { 110 + swiotlb_force = SWIOTLB_FORCE; 111 + } else if (!strcmp(str, "noforce")) { 112 + swiotlb_force = SWIOTLB_NO_FORCE; 113 + io_tlb_nslabs = 1; 114 + } 117 115 118 116 return 0; 119 117 } ··· 129 119 return io_tlb_nslabs; 130 120 } 131 121 EXPORT_SYMBOL_GPL(swiotlb_nr_tbl); 122 + 123 + unsigned int swiotlb_max_segment(void) 124 + { 125 + return max_segment; 126 + } 127 + EXPORT_SYMBOL_GPL(swiotlb_max_segment); 128 + 129 + void swiotlb_set_max_segment(unsigned int val) 130 + { 131 + if (swiotlb_force == SWIOTLB_FORCE) 132 + max_segment = 1; 133 + else 134 + max_segment = rounddown(val, PAGE_SIZE); 135 + } 132 136 133 137 /* default to 64MB */ 134 138 #define IO_TLB_DEFAULT_SIZE (64UL<<20) ··· 225 201 if (verbose) 226 202 swiotlb_print_info(); 227 203 204 + swiotlb_set_max_segment(io_tlb_nslabs << IO_TLB_SHIFT); 228 205 return 0; 229 206 } 230 207 ··· 304 279 rc = swiotlb_late_init_with_tbl(vstart, io_tlb_nslabs); 305 280 if (rc) 306 281 free_pages((unsigned long)vstart, order); 282 + 307 283 return rc; 308 284 } 309 285 ··· 359 333 360 334 late_alloc = 1; 361 335 336 + swiotlb_set_max_segment(io_tlb_nslabs << IO_TLB_SHIFT); 337 + 362 338 return 0; 363 339 364 340 cleanup4: ··· 375 347 io_tlb_end = 0; 376 348 io_tlb_start = 0; 377 349 io_tlb_nslabs = 0; 350 + max_segment = 0; 378 351 return -ENOMEM; 379 352 } 380 353 ··· 404 375 PAGE_ALIGN(io_tlb_nslabs << IO_TLB_SHIFT)); 405 376 } 406 377 io_tlb_nslabs = 0; 378 + max_segment = 0; 407 379 } 408 380 409 381 int is_swiotlb_buffer(phys_addr_t paddr) ··· 573 543 map_single(struct device *hwdev, phys_addr_t phys, size_t size, 574 544 enum dma_data_direction dir, unsigned long attrs) 575 545 { 576 - dma_addr_t start_dma_addr = phys_to_dma(hwdev, io_tlb_start); 546 + dma_addr_t start_dma_addr; 577 547 548 + if (swiotlb_force == SWIOTLB_NO_FORCE) { 549 + dev_warn_ratelimited(hwdev, "Cannot do DMA to address %pa\n", 550 + &phys); 551 + return SWIOTLB_MAP_ERROR; 552 + } 553 + 554 + start_dma_addr = phys_to_dma(hwdev, io_tlb_start); 578 555 return swiotlb_tbl_map_single(hwdev, start_dma_addr, phys, size, 579 556 dir, attrs); 580 557 } ··· 758 721 swiotlb_full(struct device *dev, size_t size, enum dma_data_direction dir, 759 722 int do_panic) 760 723 { 724 + if (swiotlb_force == SWIOTLB_NO_FORCE) 725 + return; 726 + 761 727 /* 762 728 * Ran out of IOMMU space for this operation. This is very bad. 763 729 * Unfortunately the drivers cannot handle this operation properly. ··· 803 763 * we can safely return the device addr and not worry about bounce 804 764 * buffering it. 805 765 */ 806 - if (dma_capable(dev, dev_addr, size) && !swiotlb_force) 766 + if (dma_capable(dev, dev_addr, size) && swiotlb_force != SWIOTLB_FORCE) 807 767 return dev_addr; 808 768 809 769 trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force); ··· 944 904 phys_addr_t paddr = sg_phys(sg); 945 905 dma_addr_t dev_addr = phys_to_dma(hwdev, paddr); 946 906 947 - if (swiotlb_force || 907 + if (swiotlb_force == SWIOTLB_FORCE || 948 908 !dma_capable(hwdev, dev_addr, sg->length)) { 949 909 phys_addr_t map = map_single(hwdev, sg_phys(sg), 950 910 sg->length, dir, attrs);

Configure Feed

Configure Feed