Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
nvme-pci: remove volatile cqes
The completion queue entry is not volatile once the phase is confirmed.
Remove the volatile keywords and check the phase using the appropriate
READ_ONCE() accessor, allowing the compiler to optimize the remaining
completion path.
Signed-off-by: Keith Busch <kbusch@kernel.org>
Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
Signed-off-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Keith Busch
and committed by
Jens Axboe
74943d45
b04df85d
+5
-4
+5
-4
drivers/nvme/host/pci.c
+5
-4
drivers/nvme/host/pci.c
···
166
166
void *sq_cmds;
167
167
/* only used for poll queues: */
168
168
spinlock_t cq_poll_lock ____cacheline_aligned_in_smp;
169
-
volatile struct nvme_completion *cqes;
169
+
struct nvme_completion *cqes;
170
170
dma_addr_t sq_dma_addr;
171
171
dma_addr_t cq_dma_addr;
172
172
u32 __iomem *q_db;
···
922
922
/* We read the CQE phase first to check if the rest of the entry is valid */
923
923
static inline bool nvme_cqe_pending(struct nvme_queue *nvmeq)
924
924
{
925
-
return (le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) ==
926
-
nvmeq->cq_phase;
925
+
struct nvme_completion *hcqe = &nvmeq->cqes[nvmeq->cq_head];
926
+
927
+
return (le16_to_cpu(READ_ONCE(hcqe->status)) & 1) == nvmeq->cq_phase;
927
928
}
928
929
929
930
static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq)
···
945
944
946
945
static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx)
947
946
{
948
-
volatile struct nvme_completion *cqe = &nvmeq->cqes[idx];
947
+
struct nvme_completion *cqe = &nvmeq->cqes[idx];
949
948
struct request *req;
950
949
951
950
if (unlikely(cqe->command_id >= nvmeq->q_depth)) {