nvme: apple: Add Apple A11 support · tjh.dev/kernel@04d8ecf

+137 -60

1 changed file

expand all

drivers

nvme

host

apple.c

+137 -60

drivers/nvme/host/apple.c

··· 35 35 #include "nvme.h" 36 36 37 37 #define APPLE_ANS_BOOT_TIMEOUT USEC_PER_SEC 38 - #define APPLE_ANS_MAX_QUEUE_DEPTH 64 39 38 40 39 #define APPLE_ANS_COPROC_CPU_CONTROL 0x44 41 40 #define APPLE_ANS_COPROC_CPU_CONTROL_RUN BIT(4) ··· 73 74 */ 74 75 #define APPLE_NVME_AQ_DEPTH 2 75 76 #define APPLE_NVME_AQ_MQ_TAG_DEPTH (APPLE_NVME_AQ_DEPTH - 1) 77 + 78 + #define APPLE_NVME_IOSQES 7 76 79 77 80 /* 78 81 * These can be higher, but we need to ensure that any command doesn't ··· 143 142 u32 __iomem *sq_db; 144 143 u32 __iomem *cq_db; 145 144 145 + u16 sq_tail; 146 146 u16 cq_head; 147 147 u8 cq_phase; 148 148 ··· 168 166 struct scatterlist *sg; 169 167 }; 170 168 169 + struct apple_nvme_hw { 170 + bool has_lsq_nvmmu; 171 + u32 max_queue_depth; 172 + }; 173 + 171 174 struct apple_nvme { 172 175 struct device *dev; 173 176 174 177 void __iomem *mmio_coproc; 175 178 void __iomem *mmio_nvme; 179 + const struct apple_nvme_hw *hw; 176 180 177 181 struct device **pd_dev; 178 182 struct device_link **pd_link; ··· 223 215 224 216 static unsigned int apple_nvme_queue_depth(struct apple_nvme_queue *q) 225 217 { 226 - if (q->is_adminq) 218 + struct apple_nvme *anv = queue_to_apple_nvme(q); 219 + 220 + if (q->is_adminq && anv->hw->has_lsq_nvmmu) 227 221 return APPLE_NVME_AQ_DEPTH; 228 222 229 - return APPLE_ANS_MAX_QUEUE_DEPTH; 223 + return anv->hw->max_queue_depth; 230 224 } 231 225 232 226 static void apple_nvme_rtkit_crashed(void *cookie, const void *crashlog, size_t crashlog_size) ··· 290 280 "NVMMU TCB invalidation failed\n"); 291 281 } 292 282 293 - static void apple_nvme_submit_cmd(struct apple_nvme_queue *q, 283 + static void apple_nvme_submit_cmd_t8015(struct apple_nvme_queue *q, 284 + struct nvme_command *cmd) 285 + { 286 + struct apple_nvme *anv = queue_to_apple_nvme(q); 287 + 288 + spin_lock_irq(&anv->lock); 289 + 290 + if (q->is_adminq) 291 + memcpy(&q->sqes[q->sq_tail], cmd, sizeof(*cmd)); 292 + else 293 + memcpy((void *)q->sqes + (q->sq_tail << APPLE_NVME_IOSQES), 294 + cmd, sizeof(*cmd)); 295 + 296 + if (++q->sq_tail == anv->hw->max_queue_depth) 297 + q->sq_tail = 0; 298 + 299 + writel(q->sq_tail, q->sq_db); 300 + spin_unlock_irq(&anv->lock); 301 + } 302 + 303 + 304 + static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, 294 305 struct nvme_command *cmd) 295 306 { 296 307 struct apple_nvme *anv = queue_to_apple_nvme(q); ··· 621 590 __u16 command_id = READ_ONCE(cqe->command_id); 622 591 struct request *req; 623 592 624 - apple_nvmmu_inval(q, command_id); 593 + if (anv->hw->has_lsq_nvmmu) 594 + apple_nvmmu_inval(q, command_id); 625 595 626 596 req = nvme_find_rq(apple_nvme_queue_tagset(anv, q), command_id); 627 597 if (unlikely(!req)) { ··· 717 685 c.create_cq.opcode = nvme_admin_create_cq; 718 686 c.create_cq.prp1 = cpu_to_le64(anv->ioq.cq_dma_addr); 719 687 c.create_cq.cqid = cpu_to_le16(1); 720 - c.create_cq.qsize = cpu_to_le16(APPLE_ANS_MAX_QUEUE_DEPTH - 1); 688 + c.create_cq.qsize = cpu_to_le16(anv->hw->max_queue_depth - 1); 721 689 c.create_cq.cq_flags = cpu_to_le16(NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED); 722 690 c.create_cq.irq_vector = cpu_to_le16(0); 723 691 ··· 745 713 c.create_sq.opcode = nvme_admin_create_sq; 746 714 c.create_sq.prp1 = cpu_to_le64(anv->ioq.sq_dma_addr); 747 715 c.create_sq.sqid = cpu_to_le16(1); 748 - c.create_sq.qsize = cpu_to_le16(APPLE_ANS_MAX_QUEUE_DEPTH - 1); 716 + c.create_sq.qsize = cpu_to_le16(anv->hw->max_queue_depth - 1); 749 717 c.create_sq.sq_flags = cpu_to_le16(NVME_QUEUE_PHYS_CONTIG); 750 718 c.create_sq.cqid = cpu_to_le16(1); 751 719 ··· 797 765 } 798 766 799 767 nvme_start_request(req); 800 - apple_nvme_submit_cmd(q, cmnd); 768 + 769 + if (anv->hw->has_lsq_nvmmu) 770 + apple_nvme_submit_cmd_t8103(q, cmnd); 771 + else 772 + apple_nvme_submit_cmd_t8015(q, cmnd); 773 + 801 774 return BLK_STS_OK; 802 775 803 776 out_free_cmd: ··· 1007 970 static void apple_nvme_init_queue(struct apple_nvme_queue *q) 1008 971 { 1009 972 unsigned int depth = apple_nvme_queue_depth(q); 973 + struct apple_nvme *anv = queue_to_apple_nvme(q); 1010 974 1011 975 q->cq_head = 0; 1012 976 q->cq_phase = 1; 1013 - memset(q->tcbs, 0, 1014 - APPLE_ANS_MAX_QUEUE_DEPTH * sizeof(struct apple_nvmmu_tcb)); 977 + if (anv->hw->has_lsq_nvmmu) 978 + memset(q->tcbs, 0, anv->hw->max_queue_depth 979 + * sizeof(struct apple_nvmmu_tcb)); 1015 980 memset(q->cqes, 0, depth * sizeof(struct nvme_completion)); 1016 981 WRITE_ONCE(q->enabled, true); 1017 982 wmb(); /* ensure the first interrupt sees the initialization */ ··· 1108 1069 1109 1070 dma_set_max_seg_size(anv->dev, 0xffffffff); 1110 1071 1111 - /* 1112 - * Enable NVMMU and linear submission queues. 1113 - * While we could keep those disabled and pretend this is slightly 1114 - * more common NVMe controller we'd still need some quirks (e.g. 1115 - * sq entries will be 128 bytes) and Apple might drop support for 1116 - * that mode in the future. 1117 - */ 1118 - writel(APPLE_ANS_LINEAR_SQ_EN, 1119 - anv->mmio_nvme + APPLE_ANS_LINEAR_SQ_CTRL); 1072 + if (anv->hw->has_lsq_nvmmu) { 1073 + /* 1074 + * Enable NVMMU and linear submission queues which is required 1075 + * since T6000. 1076 + */ 1077 + writel(APPLE_ANS_LINEAR_SQ_EN, 1078 + anv->mmio_nvme + APPLE_ANS_LINEAR_SQ_CTRL); 1120 1079 1121 - /* Allow as many pending command as possible for both queues */ 1122 - writel(APPLE_ANS_MAX_QUEUE_DEPTH | (APPLE_ANS_MAX_QUEUE_DEPTH << 16), 1123 - anv->mmio_nvme + APPLE_ANS_MAX_PEND_CMDS_CTRL); 1080 + /* Allow as many pending command as possible for both queues */ 1081 + writel(anv->hw->max_queue_depth 1082 + | (anv->hw->max_queue_depth << 16), anv->mmio_nvme 1083 + + APPLE_ANS_MAX_PEND_CMDS_CTRL); 1124 1084 1125 - /* Setup the NVMMU for the maximum admin and IO queue depth */ 1126 - writel(APPLE_ANS_MAX_QUEUE_DEPTH - 1, 1127 - anv->mmio_nvme + APPLE_NVMMU_NUM_TCBS); 1085 + /* Setup the NVMMU for the maximum admin and IO queue depth */ 1086 + writel(anv->hw->max_queue_depth - 1, 1087 + anv->mmio_nvme + APPLE_NVMMU_NUM_TCBS); 1128 1088 1129 - /* 1130 - * This is probably a chicken bit: without it all commands where any PRP 1131 - * is set to zero (including those that don't use that field) fail and 1132 - * the co-processor complains about "completed with err BAD_CMD-" or 1133 - * a "NULL_PRP_PTR_ERR" in the syslog 1134 - */ 1135 - writel(readl(anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL) & 1136 - ~APPLE_ANS_PRP_NULL_CHECK, 1137 - anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL); 1089 + /* 1090 + * This is probably a chicken bit: without it all commands 1091 + * where any PRP is set to zero (including those that don't use 1092 + * that field) fail and the co-processor complains about 1093 + * "completed with err BAD_CMD-" or a "NULL_PRP_PTR_ERR" in the 1094 + * syslog 1095 + */ 1096 + writel(readl(anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL) & 1097 + ~APPLE_ANS_PRP_NULL_CHECK, 1098 + anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL); 1099 + } 1138 1100 1139 1101 /* Setup the admin queue */ 1140 - aqa = APPLE_NVME_AQ_DEPTH - 1; 1102 + if (anv->hw->has_lsq_nvmmu) 1103 + aqa = APPLE_NVME_AQ_DEPTH - 1; 1104 + else 1105 + aqa = anv->hw->max_queue_depth - 1; 1141 1106 aqa |= aqa << 16; 1142 1107 writel(aqa, anv->mmio_nvme + NVME_REG_AQA); 1143 1108 writeq(anv->adminq.sq_dma_addr, anv->mmio_nvme + NVME_REG_ASQ); 1144 1109 writeq(anv->adminq.cq_dma_addr, anv->mmio_nvme + NVME_REG_ACQ); 1145 1110 1146 - /* Setup NVMMU for both queues */ 1147 - writeq(anv->adminq.tcb_dma_addr, 1148 - anv->mmio_nvme + APPLE_NVMMU_ASQ_TCB_BASE); 1149 - writeq(anv->ioq.tcb_dma_addr, 1150 - anv->mmio_nvme + APPLE_NVMMU_IOSQ_TCB_BASE); 1111 + if (anv->hw->has_lsq_nvmmu) { 1112 + /* Setup NVMMU for both queues */ 1113 + writeq(anv->adminq.tcb_dma_addr, 1114 + anv->mmio_nvme + APPLE_NVMMU_ASQ_TCB_BASE); 1115 + writeq(anv->ioq.tcb_dma_addr, 1116 + anv->mmio_nvme + APPLE_NVMMU_IOSQ_TCB_BASE); 1117 + } 1151 1118 1152 1119 anv->ctrl.sqsize = 1153 - APPLE_ANS_MAX_QUEUE_DEPTH - 1; /* 0's based queue depth */ 1120 + anv->hw->max_queue_depth - 1; /* 0's based queue depth */ 1154 1121 anv->ctrl.cap = readq(anv->mmio_nvme + NVME_REG_CAP); 1155 1122 1156 1123 dev_dbg(anv->dev, "Enabling controller now"); ··· 1327 1282 * both queues. The admin queue gets the first APPLE_NVME_AQ_DEPTH which 1328 1283 * must be marked as reserved in the IO queue. 1329 1284 */ 1330 - anv->tagset.reserved_tags = APPLE_NVME_AQ_DEPTH; 1331 - anv->tagset.queue_depth = APPLE_ANS_MAX_QUEUE_DEPTH - 1; 1285 + if (anv->hw->has_lsq_nvmmu) 1286 + anv->tagset.reserved_tags = APPLE_NVME_AQ_DEPTH; 1287 + anv->tagset.queue_depth = anv->hw->max_queue_depth - 1; 1332 1288 anv->tagset.timeout = NVME_IO_TIMEOUT; 1333 1289 anv->tagset.numa_node = NUMA_NO_NODE; 1334 1290 anv->tagset.cmd_size = sizeof(struct apple_nvme_iod); ··· 1353 1307 struct apple_nvme_queue *q) 1354 1308 { 1355 1309 unsigned int depth = apple_nvme_queue_depth(q); 1310 + size_t iosq_size; 1356 1311 1357 1312 q->cqes = dmam_alloc_coherent(anv->dev, 1358 1313 depth * sizeof(struct nvme_completion), ··· 1361 1314 if (!q->cqes) 1362 1315 return -ENOMEM; 1363 1316 1364 - q->sqes = dmam_alloc_coherent(anv->dev, 1365 - depth * sizeof(struct nvme_command), 1317 + if (anv->hw->has_lsq_nvmmu) 1318 + iosq_size = depth * sizeof(struct nvme_command); 1319 + else 1320 + iosq_size = depth << APPLE_NVME_IOSQES; 1321 + 1322 + q->sqes = dmam_alloc_coherent(anv->dev, iosq_size, 1366 1323 &q->sq_dma_addr, GFP_KERNEL); 1367 1324 if (!q->sqes) 1368 1325 return -ENOMEM; 1369 1326 1370 - /* 1371 - * We need the maximum queue depth here because the NVMMU only has a 1372 - * single depth configuration shared between both queues. 1373 - */ 1374 - q->tcbs = dmam_alloc_coherent(anv->dev, 1375 - APPLE_ANS_MAX_QUEUE_DEPTH * 1376 - sizeof(struct apple_nvmmu_tcb), 1377 - &q->tcb_dma_addr, GFP_KERNEL); 1378 - if (!q->tcbs) 1379 - return -ENOMEM; 1327 + if (anv->hw->has_lsq_nvmmu) { 1328 + /* 1329 + * We need the maximum queue depth here because the NVMMU only 1330 + * has a single depth configuration shared between both queues. 1331 + */ 1332 + q->tcbs = dmam_alloc_coherent(anv->dev, 1333 + anv->hw->max_queue_depth * 1334 + sizeof(struct apple_nvmmu_tcb), 1335 + &q->tcb_dma_addr, GFP_KERNEL); 1336 + if (!q->tcbs) 1337 + return -ENOMEM; 1338 + } 1380 1339 1381 1340 /* 1382 1341 * initialize phase to make sure the allocated and empty memory ··· 1466 1413 anv->adminq.is_adminq = true; 1467 1414 platform_set_drvdata(pdev, anv); 1468 1415 1416 + anv->hw = of_device_get_match_data(&pdev->dev); 1417 + if (!anv->hw) { 1418 + ret = -ENODEV; 1419 + goto put_dev; 1420 + } 1421 + 1469 1422 ret = apple_nvme_attach_genpd(anv); 1470 1423 if (ret < 0) { 1471 1424 dev_err_probe(dev, ret, "Failed to attach power domains"); ··· 1503 1444 goto put_dev; 1504 1445 } 1505 1446 1506 - anv->adminq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_ASQ_DB; 1507 - anv->adminq.cq_db = anv->mmio_nvme + APPLE_ANS_ACQ_DB; 1508 - anv->ioq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_IOSQ_DB; 1509 - anv->ioq.cq_db = anv->mmio_nvme + APPLE_ANS_IOCQ_DB; 1447 + if (anv->hw->has_lsq_nvmmu) { 1448 + anv->adminq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_ASQ_DB; 1449 + anv->adminq.cq_db = anv->mmio_nvme + APPLE_ANS_ACQ_DB; 1450 + anv->ioq.sq_db = anv->mmio_nvme + APPLE_ANS_LINEAR_IOSQ_DB; 1451 + anv->ioq.cq_db = anv->mmio_nvme + APPLE_ANS_IOCQ_DB; 1452 + } else { 1453 + anv->adminq.sq_db = anv->mmio_nvme + NVME_REG_DBS; 1454 + anv->adminq.cq_db = anv->mmio_nvme + APPLE_ANS_ACQ_DB; 1455 + anv->ioq.sq_db = anv->mmio_nvme + NVME_REG_DBS + 8; 1456 + anv->ioq.cq_db = anv->mmio_nvme + APPLE_ANS_IOCQ_DB; 1457 + } 1510 1458 1511 1459 anv->sart = devm_apple_sart_get(dev); 1512 1460 if (IS_ERR(anv->sart)) { ··· 1691 1625 static DEFINE_SIMPLE_DEV_PM_OPS(apple_nvme_pm_ops, apple_nvme_suspend, 1692 1626 apple_nvme_resume); 1693 1627 1628 + static const struct apple_nvme_hw apple_nvme_t8015_hw = { 1629 + .has_lsq_nvmmu = false, 1630 + .max_queue_depth = 16, 1631 + }; 1632 + 1633 + static const struct apple_nvme_hw apple_nvme_t8103_hw = { 1634 + .has_lsq_nvmmu = true, 1635 + .max_queue_depth = 64, 1636 + }; 1637 + 1694 1638 static const struct of_device_id apple_nvme_of_match[] = { 1695 - { .compatible = "apple,nvme-ans2" }, 1639 + { .compatible = "apple,t8015-nvme-ans2", .data = &apple_nvme_t8015_hw }, 1640 + { .compatible = "apple,nvme-ans2", .data = &apple_nvme_t8103_hw }, 1696 1641 {}, 1697 1642 }; 1698 1643 MODULE_DEVICE_TABLE(of, apple_nvme_of_match);

Configure Feed

Configure Feed