Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
nvme-pci: remove superfluous arguments
The call chain in the prep_rq and completion paths passes around a lot
of nvme_dev, nvme_queue and nvme_command arguments that can be trivially
derived from the passed in struct request. Remove them.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Keith Busch <kbusch@kernel.org>
Reviewed-by: Leon Romanovsky <leonro@nvidia.com>
Link: https://lore.kernel.org/r/20250625113531.522027-6-hch@lst.de
Signed-off-by: Jens Axboe <axboe@kernel.dk>
authored by
Christoph Hellwig
and committed by
Jens Axboe
deecd1c4
cd71b52a
+51
-54
+51
-54
drivers/nvme/host/pci.c
+51
-54
drivers/nvme/host/pci.c
···
584
584
SGL_FORCED,
585
585
};
586
586
587
-
static inline bool nvme_pci_metadata_use_sgls(struct nvme_dev *dev,
588
-
struct request *req)
587
+
static inline bool nvme_pci_metadata_use_sgls(struct request *req)
589
588
{
589
+
struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
590
+
struct nvme_dev *dev = nvmeq->dev;
591
+
590
592
if (!nvme_ctrl_meta_sgl_supported(&dev->ctrl))
591
593
return false;
592
594
return req->nr_integrity_segments > 1 ||
···
626
624
return nvmeq->descriptor_pools.large;
627
625
}
628
626
629
-
static void nvme_free_descriptors(struct nvme_queue *nvmeq, struct request *req)
627
+
static void nvme_free_descriptors(struct request *req)
630
628
{
629
+
struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
631
630
const int last_prp = NVME_CTRL_PAGE_SIZE / sizeof(__le64) - 1;
632
631
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
633
632
dma_addr_t dma_addr = iod->first_dma;
···
650
647
}
651
648
}
652
649
653
-
static void nvme_unmap_data(struct nvme_dev *dev, struct nvme_queue *nvmeq,
654
-
struct request *req)
650
+
static void nvme_unmap_data(struct request *req)
655
651
{
656
652
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
653
+
struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
657
654
658
655
if (iod->dma_len) {
659
-
dma_unmap_page(dev->dev, iod->first_dma, iod->dma_len,
656
+
dma_unmap_page(nvmeq->dev->dev, iod->first_dma, iod->dma_len,
660
657
rq_dma_dir(req));
661
658
return;
662
659
}
663
660
664
661
WARN_ON_ONCE(!iod->sgt.nents);
665
662
666
-
dma_unmap_sgtable(dev->dev, &iod->sgt, rq_dma_dir(req), 0);
667
-
nvme_free_descriptors(nvmeq, req);
668
-
mempool_free(iod->sgt.sgl, dev->iod_mempool);
663
+
dma_unmap_sgtable(nvmeq->dev->dev, &iod->sgt, rq_dma_dir(req), 0);
664
+
nvme_free_descriptors(req);
665
+
mempool_free(iod->sgt.sgl, nvmeq->dev->iod_mempool);
669
666
}
670
667
671
668
static void nvme_print_sgl(struct scatterlist *sgl, int nents)
···
682
679
}
683
680
}
684
681
685
-
static blk_status_t nvme_pci_setup_prps(struct nvme_queue *nvmeq,
686
-
struct request *req, struct nvme_rw_command *cmnd)
682
+
static blk_status_t nvme_pci_setup_prps(struct request *req)
687
683
{
688
684
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
685
+
struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
689
686
int length = blk_rq_payload_bytes(req);
690
687
struct scatterlist *sg = iod->sgt.sgl;
691
688
int dma_len = sg_dma_len(sg);
···
754
751
dma_len = sg_dma_len(sg);
755
752
}
756
753
done:
757
-
cmnd->dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sgt.sgl));
758
-
cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma);
754
+
iod->cmd.common.dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sgt.sgl));
755
+
iod->cmd.common.dptr.prp2 = cpu_to_le64(iod->first_dma);
759
756
return BLK_STS_OK;
760
757
free_prps:
761
-
nvme_free_descriptors(nvmeq, req);
758
+
nvme_free_descriptors(req);
762
759
return BLK_STS_RESOURCE;
763
760
bad_sgl:
764
761
WARN(DO_ONCE(nvme_print_sgl, iod->sgt.sgl, iod->sgt.nents),
···
783
780
sge->type = NVME_SGL_FMT_LAST_SEG_DESC << 4;
784
781
}
785
782
786
-
static blk_status_t nvme_pci_setup_sgls(struct nvme_queue *nvmeq,
787
-
struct request *req, struct nvme_rw_command *cmd)
783
+
static blk_status_t nvme_pci_setup_sgls(struct request *req)
788
784
{
789
785
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
786
+
struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
790
787
struct nvme_sgl_desc *sg_list;
791
788
struct scatterlist *sg = iod->sgt.sgl;
792
789
unsigned int entries = iod->sgt.nents;
···
794
791
int i = 0;
795
792
796
793
/* setting the transfer type as SGL */
797
-
cmd->flags = NVME_CMD_SGL_METABUF;
794
+
iod->cmd.common.flags = NVME_CMD_SGL_METABUF;
798
795
799
796
if (entries == 1) {
800
-
nvme_pci_sgl_set_data(&cmd->dptr.sgl, sg);
797
+
nvme_pci_sgl_set_data(&iod->cmd.common.dptr.sgl, sg);
801
798
return BLK_STS_OK;
802
799
}
803
800
···
811
808
iod->descriptors[iod->nr_descriptors++] = sg_list;
812
809
iod->first_dma = sgl_dma;
813
810
814
-
nvme_pci_sgl_set_seg(&cmd->dptr.sgl, sgl_dma, entries);
811
+
nvme_pci_sgl_set_seg(&iod->cmd.common.dptr.sgl, sgl_dma, entries);
815
812
do {
816
813
nvme_pci_sgl_set_data(&sg_list[i++], sg);
817
814
sg = sg_next(sg);
···
858
855
return BLK_STS_OK;
859
856
}
860
857
861
-
static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
862
-
struct nvme_command *cmnd)
858
+
static blk_status_t nvme_map_data(struct request *req)
863
859
{
864
-
struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
865
860
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
861
+
struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
862
+
struct nvme_dev *dev = nvmeq->dev;
866
863
enum nvme_use_sgl use_sgl = nvme_pci_use_sgls(dev, req);
867
864
blk_status_t ret = BLK_STS_RESOURCE;
868
865
int rc;
···
893
890
if (use_sgl == SGL_FORCED ||
894
891
(use_sgl == SGL_SUPPORTED &&
895
892
(sgl_threshold && nvme_pci_avg_seg_size(req) >= sgl_threshold)))
896
-
ret = nvme_pci_setup_sgls(nvmeq, req, &cmnd->rw);
893
+
ret = nvme_pci_setup_sgls(req);
897
894
else
898
-
ret = nvme_pci_setup_prps(nvmeq, req, &cmnd->rw);
895
+
ret = nvme_pci_setup_prps(req);
899
896
if (ret != BLK_STS_OK)
900
897
goto out_unmap_sg;
901
898
return BLK_STS_OK;
···
907
904
return ret;
908
905
}
909
906
910
-
static blk_status_t nvme_pci_setup_meta_sgls(struct nvme_dev *dev,
911
-
struct request *req)
907
+
static blk_status_t nvme_pci_setup_meta_sgls(struct request *req)
912
908
{
913
909
struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
910
+
struct nvme_dev *dev = nvmeq->dev;
914
911
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
915
-
struct nvme_rw_command *cmnd = &iod->cmd.rw;
916
912
struct nvme_sgl_desc *sg_list;
917
913
struct scatterlist *sgl, *sg;
918
914
unsigned int entries;
···
942
940
iod->meta_descriptor = sg_list;
943
941
iod->meta_dma = sgl_dma;
944
942
945
-
cmnd->flags = NVME_CMD_SGL_METASEG;
946
-
cmnd->metadata = cpu_to_le64(sgl_dma);
943
+
iod->cmd.common.flags = NVME_CMD_SGL_METASEG;
944
+
iod->cmd.common.metadata = cpu_to_le64(sgl_dma);
947
945
948
946
sgl = iod->meta_sgt.sgl;
949
947
if (entries == 1) {
···
965
963
return BLK_STS_RESOURCE;
966
964
}
967
965
968
-
static blk_status_t nvme_pci_setup_meta_mptr(struct nvme_dev *dev,
969
-
struct request *req)
966
+
static blk_status_t nvme_pci_setup_meta_mptr(struct request *req)
970
967
{
971
968
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
969
+
struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
972
970
struct bio_vec bv = rq_integrity_vec(req);
973
-
struct nvme_command *cmnd = &iod->cmd;
974
971
975
-
iod->meta_dma = dma_map_bvec(dev->dev, &bv, rq_dma_dir(req), 0);
976
-
if (dma_mapping_error(dev->dev, iod->meta_dma))
972
+
iod->meta_dma = dma_map_bvec(nvmeq->dev->dev, &bv, rq_dma_dir(req), 0);
973
+
if (dma_mapping_error(nvmeq->dev->dev, iod->meta_dma))
977
974
return BLK_STS_IOERR;
978
-
cmnd->rw.metadata = cpu_to_le64(iod->meta_dma);
975
+
iod->cmd.common.metadata = cpu_to_le64(iod->meta_dma);
979
976
return BLK_STS_OK;
980
977
}
981
978
982
-
static blk_status_t nvme_map_metadata(struct nvme_dev *dev, struct request *req)
979
+
static blk_status_t nvme_map_metadata(struct request *req)
983
980
{
984
981
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
985
982
986
983
if ((iod->cmd.common.flags & NVME_CMD_SGL_METABUF) &&
987
-
nvme_pci_metadata_use_sgls(dev, req))
988
-
return nvme_pci_setup_meta_sgls(dev, req);
989
-
return nvme_pci_setup_meta_mptr(dev, req);
984
+
nvme_pci_metadata_use_sgls(req))
985
+
return nvme_pci_setup_meta_sgls(req);
986
+
return nvme_pci_setup_meta_mptr(req);
990
987
}
991
988
992
-
static blk_status_t nvme_prep_rq(struct nvme_dev *dev, struct request *req)
989
+
static blk_status_t nvme_prep_rq(struct request *req)
993
990
{
994
991
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
995
992
blk_status_t ret;
···
1003
1002
return ret;
1004
1003
1005
1004
if (blk_rq_nr_phys_segments(req)) {
1006
-
ret = nvme_map_data(dev, req, &iod->cmd);
1005
+
ret = nvme_map_data(req);
1007
1006
if (ret)
1008
1007
goto out_free_cmd;
1009
1008
}
1010
1009
1011
1010
if (blk_integrity_rq(req)) {
1012
-
ret = nvme_map_metadata(dev, req);
1011
+
ret = nvme_map_metadata(req);
1013
1012
if (ret)
1014
1013
goto out_unmap_data;
1015
1014
}
···
1018
1017
return BLK_STS_OK;
1019
1018
out_unmap_data:
1020
1019
if (blk_rq_nr_phys_segments(req))
1021
-
nvme_unmap_data(dev, req->mq_hctx->driver_data, req);
1020
+
nvme_unmap_data(req);
1022
1021
out_free_cmd:
1023
1022
nvme_cleanup_cmd(req);
1024
1023
return ret;
···
1043
1042
if (unlikely(!nvme_check_ready(&dev->ctrl, req, true)))
1044
1043
return nvme_fail_nonready_command(&dev->ctrl, req);
1045
1044
1046
-
ret = nvme_prep_rq(dev, req);
1045
+
ret = nvme_prep_rq(req);
1047
1046
if (unlikely(ret))
1048
1047
return ret;
1049
1048
spin_lock(&nvmeq->sq_lock);
···
1081
1080
if (unlikely(!nvme_check_ready(&nvmeq->dev->ctrl, req, true)))
1082
1081
return false;
1083
1082
1084
-
return nvme_prep_rq(nvmeq->dev, req) == BLK_STS_OK;
1083
+
return nvme_prep_rq(req) == BLK_STS_OK;
1085
1084
}
1086
1085
1087
1086
static void nvme_queue_rqs(struct rq_list *rqlist)
···
1107
1106
*rqlist = requeue_list;
1108
1107
}
1109
1108
1110
-
static __always_inline void nvme_unmap_metadata(struct nvme_dev *dev,
1111
-
struct nvme_queue *nvmeq,
1112
-
struct request *req)
1109
+
static __always_inline void nvme_unmap_metadata(struct request *req)
1113
1110
{
1114
1111
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
1112
+
struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
1113
+
struct nvme_dev *dev = nvmeq->dev;
1115
1114
1116
1115
if (!iod->meta_sgt.nents) {
1117
1116
dma_unmap_page(dev->dev, iod->meta_dma,
···
1128
1127
1129
1128
static __always_inline void nvme_pci_unmap_rq(struct request *req)
1130
1129
{
1131
-
struct nvme_queue *nvmeq = req->mq_hctx->driver_data;
1132
-
struct nvme_dev *dev = nvmeq->dev;
1133
-
1134
1130
if (blk_integrity_rq(req))
1135
-
nvme_unmap_metadata(dev, nvmeq, req);
1136
-
1131
+
nvme_unmap_metadata(req);
1137
1132
if (blk_rq_nr_phys_segments(req))
1138
-
nvme_unmap_data(dev, nvmeq, req);
1133
+
nvme_unmap_data(req);
1139
1134
}
1140
1135
1141
1136
static void nvme_pci_complete_rq(struct request *req)