Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'dma-mapping-5.3-1' of git://git.infradead.org/users/hch/dma-mapping
Pull dma-mapping fixes from Christoph Hellwig:
"Fix various regressions:
- force unencrypted dma-coherent buffers if encryption bit can't fit
into the dma coherent mask (Tom Lendacky)
- avoid limiting request size if swiotlb is not used (me)
- fix swiotlb handling in dma_direct_sync_sg_for_cpu/device (Fugang
Duan)"
* tag 'dma-mapping-5.3-1' of git://git.infradead.org/users/hch/dma-mapping:
dma-direct: correct the physical addr in dma_direct_sync_sg_for_cpu/device
dma-direct: only limit the mapping size if swiotlb could be used
dma-mapping: add a dma_addressing_limited helper
dma-direct: Force unencrypted DMA under SME for certain DMA masks
+83
-26
+1
arch/s390/Kconfig
+1
arch/s390/Kconfig
+6
-1
arch/s390/mm/init.c
+6
-1
arch/s390/mm/init.c
···
30
30
#include <linux/export.h>
31
31
#include <linux/cma.h>
32
32
#include <linux/gfp.h>
33
-
#include <linux/dma-mapping.h>
33
+
#include <linux/dma-direct.h>
34
34
#include <asm/processor.h>
35
35
#include <linux/uaccess.h>
36
36
#include <asm/pgtable.h>
···
159
159
bool sev_active(void)
160
160
{
161
161
return is_prot_virt_guest();
162
+
}
163
+
164
+
bool force_dma_unencrypted(struct device *dev)
165
+
{
166
+
return sev_active();
162
167
}
163
168
164
169
/* protected virtualization */
+1
arch/x86/Kconfig
+1
arch/x86/Kconfig
···
1526
1526
depends on X86_64 && CPU_SUP_AMD
1527
1527
select DYNAMIC_PHYSICAL_MASK
1528
1528
select ARCH_USE_MEMREMAP_PROT
1529
+
select ARCH_HAS_FORCE_DMA_UNENCRYPTED
1529
1530
---help---
1530
1531
Say yes to enable support for the encryption of system memory.
1531
1532
This requires an AMD processor that supports Secure Memory
+30
arch/x86/mm/mem_encrypt.c
+30
arch/x86/mm/mem_encrypt.c
···
15
15
#include <linux/dma-direct.h>
16
16
#include <linux/swiotlb.h>
17
17
#include <linux/mem_encrypt.h>
18
+
#include <linux/device.h>
19
+
#include <linux/kernel.h>
20
+
#include <linux/bitops.h>
21
+
#include <linux/dma-mapping.h>
18
22
19
23
#include <asm/tlbflush.h>
20
24
#include <asm/fixmap.h>
···
351
347
return sme_me_mask && sev_enabled;
352
348
}
353
349
EXPORT_SYMBOL(sev_active);
350
+
351
+
/* Override for DMA direct allocation check - ARCH_HAS_FORCE_DMA_UNENCRYPTED */
352
+
bool force_dma_unencrypted(struct device *dev)
353
+
{
354
+
/*
355
+
* For SEV, all DMA must be to unencrypted addresses.
356
+
*/
357
+
if (sev_active())
358
+
return true;
359
+
360
+
/*
361
+
* For SME, all DMA must be to unencrypted addresses if the
362
+
* device does not support DMA to addresses that include the
363
+
* encryption mask.
364
+
*/
365
+
if (sme_active()) {
366
+
u64 dma_enc_mask = DMA_BIT_MASK(__ffs64(sme_me_mask));
367
+
u64 dma_dev_mask = min_not_zero(dev->coherent_dma_mask,
368
+
dev->bus_dma_mask);
369
+
370
+
if (dma_dev_mask <= dma_enc_mask)
371
+
return true;
372
+
}
373
+
374
+
return false;
375
+
}
354
376
355
377
/* Architecture __weak replacement functions */
356
378
void __init mem_encrypt_free_decrypted_mem(void)
+9
include/linux/dma-direct.h
+9
include/linux/dma-direct.h
···
32
32
}
33
33
#endif /* !CONFIG_ARCH_HAS_PHYS_TO_DMA */
34
34
35
+
#ifdef CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED
36
+
bool force_dma_unencrypted(struct device *dev);
37
+
#else
38
+
static inline bool force_dma_unencrypted(struct device *dev)
39
+
{
40
+
return false;
41
+
}
42
+
#endif /* CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED */
43
+
35
44
/*
36
45
* If memory encryption is supported, phys_to_dma will set the memory encryption
37
46
* bit in the DMA address, and dma_to_phys will clear it. The raw __phys_to_dma
+14
include/linux/dma-mapping.h
+14
include/linux/dma-mapping.h
···
679
679
return dma_set_mask_and_coherent(dev, mask);
680
680
}
681
681
682
+
/**
683
+
* dma_addressing_limited - return if the device is addressing limited
684
+
* @dev: device to check
685
+
*
686
+
* Return %true if the devices DMA mask is too small to address all memory in
687
+
* the system, else %false. Lack of addressing bits is the prime reason for
688
+
* bounce buffering, but might not be the only one.
689
+
*/
690
+
static inline bool dma_addressing_limited(struct device *dev)
691
+
{
692
+
return min_not_zero(*dev->dma_mask, dev->bus_dma_mask) <
693
+
dma_get_required_mask(dev);
694
+
}
695
+
682
696
#ifdef CONFIG_ARCH_HAS_SETUP_DMA_OPS
683
697
void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
684
698
const struct iommu_ops *iommu, bool coherent);
+3
kernel/dma/Kconfig
+3
kernel/dma/Kconfig
+19
-25
kernel/dma/direct.c
+19
-25
kernel/dma/direct.c
···
23
23
#define ARCH_ZONE_DMA_BITS 24
24
24
#endif
25
25
26
-
/*
27
-
* For AMD SEV all DMA must be to unencrypted addresses.
28
-
*/
29
-
static inline bool force_dma_unencrypted(void)
30
-
{
31
-
return sev_active();
32
-
}
33
-
34
26
static void report_addr(struct device *dev, dma_addr_t dma_addr, size_t size)
35
27
{
36
28
if (!dev->dma_mask) {
···
38
46
static inline dma_addr_t phys_to_dma_direct(struct device *dev,
39
47
phys_addr_t phys)
40
48
{
41
-
if (force_dma_unencrypted())
49
+
if (force_dma_unencrypted(dev))
42
50
return __phys_to_dma(dev, phys);
43
51
return phys_to_dma(dev, phys);
44
52
}
···
59
67
if (dev->bus_dma_mask && dev->bus_dma_mask < dma_mask)
60
68
dma_mask = dev->bus_dma_mask;
61
69
62
-
if (force_dma_unencrypted())
70
+
if (force_dma_unencrypted(dev))
63
71
*phys_mask = __dma_to_phys(dev, dma_mask);
64
72
else
65
73
*phys_mask = dma_to_phys(dev, dma_mask);
···
151
159
}
152
160
153
161
ret = page_address(page);
154
-
if (force_dma_unencrypted()) {
162
+
if (force_dma_unencrypted(dev)) {
155
163
set_memory_decrypted((unsigned long)ret, 1 << get_order(size));
156
164
*dma_handle = __phys_to_dma(dev, page_to_phys(page));
157
165
} else {
···
184
192
return;
185
193
}
186
194
187
-
if (force_dma_unencrypted())
195
+
if (force_dma_unencrypted(dev))
188
196
set_memory_encrypted((unsigned long)cpu_addr, 1 << page_order);
189
197
190
198
if (IS_ENABLED(CONFIG_ARCH_HAS_UNCACHED_SEGMENT) &&
···
234
242
int i;
235
243
236
244
for_each_sg(sgl, sg, nents, i) {
237
-
if (unlikely(is_swiotlb_buffer(sg_phys(sg))))
238
-
swiotlb_tbl_sync_single(dev, sg_phys(sg), sg->length,
245
+
phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg));
246
+
247
+
if (unlikely(is_swiotlb_buffer(paddr)))
248
+
swiotlb_tbl_sync_single(dev, paddr, sg->length,
239
249
dir, SYNC_FOR_DEVICE);
240
250
241
251
if (!dev_is_dma_coherent(dev))
242
-
arch_sync_dma_for_device(dev, sg_phys(sg), sg->length,
252
+
arch_sync_dma_for_device(dev, paddr, sg->length,
243
253
dir);
244
254
}
245
255
}
···
273
279
int i;
274
280
275
281
for_each_sg(sgl, sg, nents, i) {
282
+
phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg));
283
+
276
284
if (!dev_is_dma_coherent(dev))
277
-
arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir);
278
-
279
-
if (unlikely(is_swiotlb_buffer(sg_phys(sg))))
280
-
swiotlb_tbl_sync_single(dev, sg_phys(sg), sg->length, dir,
285
+
arch_sync_dma_for_cpu(dev, paddr, sg->length, dir);
286
+
287
+
if (unlikely(is_swiotlb_buffer(paddr)))
288
+
swiotlb_tbl_sync_single(dev, paddr, sg->length, dir,
281
289
SYNC_FOR_CPU);
282
290
}
283
291
···
403
407
404
408
size_t dma_direct_max_mapping_size(struct device *dev)
405
409
{
406
-
size_t size = SIZE_MAX;
407
-
408
410
/* If SWIOTLB is active, use its maximum mapping size */
409
-
if (is_swiotlb_active())
410
-
size = swiotlb_max_mapping_size(dev);
411
-
412
-
return size;
411
+
if (is_swiotlb_active() &&
412
+
(dma_addressing_limited(dev) || swiotlb_force == SWIOTLB_FORCE))
413
+
return swiotlb_max_mapping_size(dev);
414
+
return SIZE_MAX;
413
415
}