tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

iommu/amd: serialize sequence allocation under concurrent TLB invalidations

With concurrent TLB invalidations, completion wait randomly gets timed out
because cmd_sem_val was incremented outside the IOMMU spinlock, allowing
CMD_COMPL_WAIT commands to be queued out of sequence and breaking the
ordering assumption in wait_on_sem().
Move the cmd_sem_val increment under iommu->lock so completion sequence
allocation is serialized with command queuing.
And remove the unnecessary return.

Fixes: d2a0cac10597 ("iommu/amd: move wait_on_sem() out of spinlock")

Tested-by: Srikanth Aithal <sraithal@amd.com>
Reported-by: Srikanth Aithal <sraithal@amd.com>
Signed-off-by: Ankit Soni <Ankit.Soni@amd.com>
Reviewed-by: Vasant Hegde <vasant.hegde@amd.com>
Signed-off-by: Joerg Roedel <joerg.roedel@amd.com>

authored by

Ankit Soni and committed by
Joerg Roedel 9e249c48 5b0530bb

+14 -8
+1 -1
drivers/iommu/amd/amd_iommu_types.h
··· 752 752 753 753 u32 flags; 754 754 volatile u64 *cmd_sem; 755 - atomic64_t cmd_sem_val; 755 + u64 cmd_sem_val; 756 756 /* 757 757 * Track physical address to directly use it in build_completion_wait() 758 758 * and avoid adding any special checks and handling for kdump.
+1 -1
drivers/iommu/amd/init.c
··· 1885 1885 iommu->pci_seg = pci_seg; 1886 1886 1887 1887 raw_spin_lock_init(&iommu->lock); 1888 - atomic64_set(&iommu->cmd_sem_val, 0); 1888 + iommu->cmd_sem_val = 0; 1889 1889 1890 1890 /* Add IOMMU to internal data structures */ 1891 1891 list_add_tail(&iommu->list, &amd_iommu_list);
+12 -6
drivers/iommu/amd/iommu.c
··· 1439 1439 return iommu_queue_command_sync(iommu, cmd, true); 1440 1440 } 1441 1441 1442 + static u64 get_cmdsem_val(struct amd_iommu *iommu) 1443 + { 1444 + lockdep_assert_held(&iommu->lock); 1445 + return ++iommu->cmd_sem_val; 1446 + } 1447 + 1442 1448 /* 1443 1449 * This function queues a completion wait command into the command 1444 1450 * buffer of an IOMMU ··· 1459 1453 if (!iommu->need_sync) 1460 1454 return 0; 1461 1455 1462 - data = atomic64_inc_return(&iommu->cmd_sem_val); 1463 - build_completion_wait(&cmd, iommu, data); 1464 - 1465 1456 raw_spin_lock_irqsave(&iommu->lock, flags); 1457 + 1458 + data = get_cmdsem_val(iommu); 1459 + build_completion_wait(&cmd, iommu, data); 1466 1460 1467 1461 ret = __iommu_queue_command_sync(iommu, &cmd, false); 1468 1462 raw_spin_unlock_irqrestore(&iommu->lock, flags); ··· 3183 3177 return; 3184 3178 3185 3179 build_inv_irt(&cmd, devid); 3186 - data = atomic64_inc_return(&iommu->cmd_sem_val); 3187 - build_completion_wait(&cmd2, iommu, data); 3188 3180 3189 3181 raw_spin_lock_irqsave(&iommu->lock, flags); 3182 + data = get_cmdsem_val(iommu); 3183 + build_completion_wait(&cmd2, iommu, data); 3184 + 3190 3185 ret = __iommu_queue_command_sync(iommu, &cmd, true); 3191 3186 if (ret) 3192 3187 goto out_err; ··· 3201 3194 3202 3195 out_err: 3203 3196 raw_spin_unlock_irqrestore(&iommu->lock, flags); 3204 - return; 3205 3197 } 3206 3198 3207 3199 static inline u8 iommu_get_int_tablen(struct iommu_dev_data *dev_data)