Merge tag 'ntb-4.5' of git://github.com/jonmason/ntb

+6

MAINTAINERS

··· 7702 7702 T: git git://github.com/jonmason/ntb.git 7703 7703 F: drivers/ntb/hw/intel/ 7704 7704 7705 + NTB AMD DRIVER 7706 + M: Xiangliang Yu <Xiangliang.Yu@amd.com> 7707 + L: linux-ntb@googlegroups.com 7708 + S: Supported 7709 + F: drivers/ntb/hw/amd/ 7710 + 7705 7711 NTFS FILESYSTEM 7706 7712 M: Anton Altaparmakov <anton@tuxera.com> 7707 7713 L: linux-ntfs-dev@lists.sourceforge.net

+1

drivers/ntb/hw/Kconfig

··· 1 + source "drivers/ntb/hw/amd/Kconfig" 1 2 source "drivers/ntb/hw/intel/Kconfig"

+1

drivers/ntb/hw/Makefile

··· 1 + obj-$(CONFIG_NTB_AMD) += amd/ 1 2 obj-$(CONFIG_NTB_INTEL) += intel/

+7

drivers/ntb/hw/amd/Kconfig

··· 1 + config NTB_AMD 2 + tristate "AMD Non-Transparent Bridge support" 3 + depends on X86_64 4 + help 5 + This driver supports AMD NTB on capable Zeppelin hardware. 6 + 7 + If unsure, say N.

+1

drivers/ntb/hw/amd/Makefile

··· 1 + obj-$(CONFIG_NTB_AMD) += ntb_hw_amd.o

+1143

drivers/ntb/hw/amd/ntb_hw_amd.c

··· 1 + /* 2 + * This file is provided under a dual BSD/GPLv2 license. When using or 3 + * redistributing this file, you may do so under either license. 4 + * 5 + * GPL LICENSE SUMMARY 6 + * 7 + * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved. 8 + * 9 + * This program is free software; you can redistribute it and/or modify 10 + * it under the terms of version 2 of the GNU General Public License as 11 + * published by the Free Software Foundation. 12 + * 13 + * BSD LICENSE 14 + * 15 + * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved. 16 + * 17 + * Redistribution and use in source and binary forms, with or without 18 + * modification, are permitted provided that the following conditions 19 + * are met: 20 + * 21 + * * Redistributions of source code must retain the above copyright 22 + * notice, this list of conditions and the following disclaimer. 23 + * * Redistributions in binary form must reproduce the above copy 24 + * notice, this list of conditions and the following disclaimer in 25 + * the documentation and/or other materials provided with the 26 + * distribution. 27 + * * Neither the name of AMD Corporation nor the names of its 28 + * contributors may be used to endorse or promote products derived 29 + * from this software without specific prior written permission. 30 + * 31 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 32 + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 33 + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 34 + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 35 + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 36 + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 37 + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 38 + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 39 + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 40 + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 41 + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 42 + * 43 + * AMD PCIe NTB Linux driver 44 + * 45 + * Contact Information: 46 + * Xiangliang Yu <Xiangliang.Yu@amd.com> 47 + */ 48 + 49 + #include <linux/debugfs.h> 50 + #include <linux/delay.h> 51 + #include <linux/init.h> 52 + #include <linux/interrupt.h> 53 + #include <linux/module.h> 54 + #include <linux/acpi.h> 55 + #include <linux/pci.h> 56 + #include <linux/random.h> 57 + #include <linux/slab.h> 58 + #include <linux/ntb.h> 59 + 60 + #include "ntb_hw_amd.h" 61 + 62 + #define NTB_NAME "ntb_hw_amd" 63 + #define NTB_DESC "AMD(R) PCI-E Non-Transparent Bridge Driver" 64 + #define NTB_VER "1.0" 65 + 66 + MODULE_DESCRIPTION(NTB_DESC); 67 + MODULE_VERSION(NTB_VER); 68 + MODULE_LICENSE("Dual BSD/GPL"); 69 + MODULE_AUTHOR("AMD Inc."); 70 + 71 + static const struct file_operations amd_ntb_debugfs_info; 72 + static struct dentry *debugfs_dir; 73 + 74 + static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx) 75 + { 76 + if (idx < 0 || idx > ndev->mw_count) 77 + return -EINVAL; 78 + 79 + return 1 << idx; 80 + } 81 + 82 + static int amd_ntb_mw_count(struct ntb_dev *ntb) 83 + { 84 + return ntb_ndev(ntb)->mw_count; 85 + } 86 + 87 + static int amd_ntb_mw_get_range(struct ntb_dev *ntb, int idx, 88 + phys_addr_t *base, 89 + resource_size_t *size, 90 + resource_size_t *align, 91 + resource_size_t *align_size) 92 + { 93 + struct amd_ntb_dev *ndev = ntb_ndev(ntb); 94 + int bar; 95 + 96 + bar = ndev_mw_to_bar(ndev, idx); 97 + if (bar < 0) 98 + return bar; 99 + 100 + if (base) 101 + *base = pci_resource_start(ndev->ntb.pdev, bar); 102 + 103 + if (size) 104 + *size = pci_resource_len(ndev->ntb.pdev, bar); 105 + 106 + if (align) 107 + *align = SZ_4K; 108 + 109 + if (align_size) 110 + *align_size = 1; 111 + 112 + return 0; 113 + } 114 + 115 + static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int idx, 116 + dma_addr_t addr, resource_size_t size) 117 + { 118 + struct amd_ntb_dev *ndev = ntb_ndev(ntb); 119 + unsigned long xlat_reg, limit_reg = 0; 120 + resource_size_t mw_size; 121 + void __iomem *mmio, *peer_mmio; 122 + u64 base_addr, limit, reg_val; 123 + int bar; 124 + 125 + bar = ndev_mw_to_bar(ndev, idx); 126 + if (bar < 0) 127 + return bar; 128 + 129 + mw_size = pci_resource_len(ndev->ntb.pdev, bar); 130 + 131 + /* make sure the range fits in the usable mw size */ 132 + if (size > mw_size) 133 + return -EINVAL; 134 + 135 + mmio = ndev->self_mmio; 136 + peer_mmio = ndev->peer_mmio; 137 + 138 + base_addr = pci_resource_start(ndev->ntb.pdev, bar); 139 + 140 + if (bar != 1) { 141 + xlat_reg = AMD_BAR23XLAT_OFFSET + ((bar - 2) << 3); 142 + limit_reg = AMD_BAR23LMT_OFFSET + ((bar - 2) << 3); 143 + 144 + /* Set the limit if supported */ 145 + limit = base_addr + size; 146 + 147 + /* set and verify setting the translation address */ 148 + write64(addr, peer_mmio + xlat_reg); 149 + reg_val = read64(peer_mmio + xlat_reg); 150 + if (reg_val != addr) { 151 + write64(0, peer_mmio + xlat_reg); 152 + return -EIO; 153 + } 154 + 155 + /* set and verify setting the limit */ 156 + write64(limit, mmio + limit_reg); 157 + reg_val = read64(mmio + limit_reg); 158 + if (reg_val != limit) { 159 + write64(base_addr, mmio + limit_reg); 160 + write64(0, peer_mmio + xlat_reg); 161 + return -EIO; 162 + } 163 + } else { 164 + xlat_reg = AMD_BAR1XLAT_OFFSET; 165 + limit_reg = AMD_BAR1LMT_OFFSET; 166 + 167 + /* split bar addr range must all be 32 bit */ 168 + if (addr & (~0ull << 32)) 169 + return -EINVAL; 170 + if ((addr + size) & (~0ull << 32)) 171 + return -EINVAL; 172 + 173 + /* Set the limit if supported */ 174 + limit = base_addr + size; 175 + 176 + /* set and verify setting the translation address */ 177 + write64(addr, peer_mmio + xlat_reg); 178 + reg_val = read64(peer_mmio + xlat_reg); 179 + if (reg_val != addr) { 180 + write64(0, peer_mmio + xlat_reg); 181 + return -EIO; 182 + } 183 + 184 + /* set and verify setting the limit */ 185 + writel(limit, mmio + limit_reg); 186 + reg_val = readl(mmio + limit_reg); 187 + if (reg_val != limit) { 188 + writel(base_addr, mmio + limit_reg); 189 + writel(0, peer_mmio + xlat_reg); 190 + return -EIO; 191 + } 192 + } 193 + 194 + return 0; 195 + } 196 + 197 + static int amd_link_is_up(struct amd_ntb_dev *ndev) 198 + { 199 + if (!ndev->peer_sta) 200 + return NTB_LNK_STA_ACTIVE(ndev->cntl_sta); 201 + 202 + /* If peer_sta is reset or D0 event, the ISR has 203 + * started a timer to check link status of hardware. 204 + * So here just clear status bit. And if peer_sta is 205 + * D3 or PME_TO, D0/reset event will be happened when 206 + * system wakeup/poweron, so do nothing here. 207 + */ 208 + if (ndev->peer_sta & AMD_PEER_RESET_EVENT) 209 + ndev->peer_sta &= ~AMD_PEER_RESET_EVENT; 210 + else if (ndev->peer_sta & AMD_PEER_D0_EVENT) 211 + ndev->peer_sta = 0; 212 + 213 + return 0; 214 + } 215 + 216 + static int amd_ntb_link_is_up(struct ntb_dev *ntb, 217 + enum ntb_speed *speed, 218 + enum ntb_width *width) 219 + { 220 + struct amd_ntb_dev *ndev = ntb_ndev(ntb); 221 + int ret = 0; 222 + 223 + if (amd_link_is_up(ndev)) { 224 + if (speed) 225 + *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta); 226 + if (width) 227 + *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta); 228 + 229 + dev_dbg(ndev_dev(ndev), "link is up.\n"); 230 + 231 + ret = 1; 232 + } else { 233 + if (speed) 234 + *speed = NTB_SPEED_NONE; 235 + if (width) 236 + *width = NTB_WIDTH_NONE; 237 + 238 + dev_dbg(ndev_dev(ndev), "link is down.\n"); 239 + } 240 + 241 + return ret; 242 + } 243 + 244 + static int amd_ntb_link_enable(struct ntb_dev *ntb, 245 + enum ntb_speed max_speed, 246 + enum ntb_width max_width) 247 + { 248 + struct amd_ntb_dev *ndev = ntb_ndev(ntb); 249 + void __iomem *mmio = ndev->self_mmio; 250 + u32 ntb_ctl; 251 + 252 + /* Enable event interrupt */ 253 + ndev->int_mask &= ~AMD_EVENT_INTMASK; 254 + writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); 255 + 256 + if (ndev->ntb.topo == NTB_TOPO_SEC) 257 + return -EINVAL; 258 + dev_dbg(ndev_dev(ndev), "Enabling Link.\n"); 259 + 260 + ntb_ctl = readl(mmio + AMD_CNTL_OFFSET); 261 + ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL); 262 + writel(ntb_ctl, mmio + AMD_CNTL_OFFSET); 263 + 264 + return 0; 265 + } 266 + 267 + static int amd_ntb_link_disable(struct ntb_dev *ntb) 268 + { 269 + struct amd_ntb_dev *ndev = ntb_ndev(ntb); 270 + void __iomem *mmio = ndev->self_mmio; 271 + u32 ntb_ctl; 272 + 273 + /* Disable event interrupt */ 274 + ndev->int_mask |= AMD_EVENT_INTMASK; 275 + writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); 276 + 277 + if (ndev->ntb.topo == NTB_TOPO_SEC) 278 + return -EINVAL; 279 + dev_dbg(ndev_dev(ndev), "Enabling Link.\n"); 280 + 281 + ntb_ctl = readl(mmio + AMD_CNTL_OFFSET); 282 + ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL); 283 + writel(ntb_ctl, mmio + AMD_CNTL_OFFSET); 284 + 285 + return 0; 286 + } 287 + 288 + static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb) 289 + { 290 + return ntb_ndev(ntb)->db_valid_mask; 291 + } 292 + 293 + static int amd_ntb_db_vector_count(struct ntb_dev *ntb) 294 + { 295 + return ntb_ndev(ntb)->db_count; 296 + } 297 + 298 + static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) 299 + { 300 + struct amd_ntb_dev *ndev = ntb_ndev(ntb); 301 + 302 + if (db_vector < 0 || db_vector > ndev->db_count) 303 + return 0; 304 + 305 + return ntb_ndev(ntb)->db_valid_mask & (1 << db_vector); 306 + } 307 + 308 + static u64 amd_ntb_db_read(struct ntb_dev *ntb) 309 + { 310 + struct amd_ntb_dev *ndev = ntb_ndev(ntb); 311 + void __iomem *mmio = ndev->self_mmio; 312 + 313 + return (u64)readw(mmio + AMD_DBSTAT_OFFSET); 314 + } 315 + 316 + static int amd_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) 317 + { 318 + struct amd_ntb_dev *ndev = ntb_ndev(ntb); 319 + void __iomem *mmio = ndev->self_mmio; 320 + 321 + writew((u16)db_bits, mmio + AMD_DBSTAT_OFFSET); 322 + 323 + return 0; 324 + } 325 + 326 + static int amd_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) 327 + { 328 + struct amd_ntb_dev *ndev = ntb_ndev(ntb); 329 + void __iomem *mmio = ndev->self_mmio; 330 + unsigned long flags; 331 + 332 + if (db_bits & ~ndev->db_valid_mask) 333 + return -EINVAL; 334 + 335 + spin_lock_irqsave(&ndev->db_mask_lock, flags); 336 + ndev->db_mask |= db_bits; 337 + writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET); 338 + spin_unlock_irqrestore(&ndev->db_mask_lock, flags); 339 + 340 + return 0; 341 + } 342 + 343 + static int amd_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) 344 + { 345 + struct amd_ntb_dev *ndev = ntb_ndev(ntb); 346 + void __iomem *mmio = ndev->self_mmio; 347 + unsigned long flags; 348 + 349 + if (db_bits & ~ndev->db_valid_mask) 350 + return -EINVAL; 351 + 352 + spin_lock_irqsave(&ndev->db_mask_lock, flags); 353 + ndev->db_mask &= ~db_bits; 354 + writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET); 355 + spin_unlock_irqrestore(&ndev->db_mask_lock, flags); 356 + 357 + return 0; 358 + } 359 + 360 + static int amd_ntb_peer_db_addr(struct ntb_dev *ntb, 361 + phys_addr_t *db_addr, 362 + resource_size_t *db_size) 363 + { 364 + struct amd_ntb_dev *ndev = ntb_ndev(ntb); 365 + 366 + if (db_addr) 367 + *db_addr = (phys_addr_t)(ndev->peer_mmio + AMD_DBREQ_OFFSET); 368 + if (db_size) 369 + *db_size = sizeof(u32); 370 + 371 + return 0; 372 + } 373 + 374 + static int amd_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) 375 + { 376 + struct amd_ntb_dev *ndev = ntb_ndev(ntb); 377 + void __iomem *mmio = ndev->self_mmio; 378 + 379 + writew((u16)db_bits, mmio + AMD_DBREQ_OFFSET); 380 + 381 + return 0; 382 + } 383 + 384 + static int amd_ntb_spad_count(struct ntb_dev *ntb) 385 + { 386 + return ntb_ndev(ntb)->spad_count; 387 + } 388 + 389 + static u32 amd_ntb_spad_read(struct ntb_dev *ntb, int idx) 390 + { 391 + struct amd_ntb_dev *ndev = ntb_ndev(ntb); 392 + void __iomem *mmio = ndev->self_mmio; 393 + u32 offset; 394 + 395 + if (idx < 0 || idx >= ndev->spad_count) 396 + return 0; 397 + 398 + offset = ndev->self_spad + (idx << 2); 399 + return readl(mmio + AMD_SPAD_OFFSET + offset); 400 + } 401 + 402 + static int amd_ntb_spad_write(struct ntb_dev *ntb, 403 + int idx, u32 val) 404 + { 405 + struct amd_ntb_dev *ndev = ntb_ndev(ntb); 406 + void __iomem *mmio = ndev->self_mmio; 407 + u32 offset; 408 + 409 + if (idx < 0 || idx >= ndev->spad_count) 410 + return -EINVAL; 411 + 412 + offset = ndev->self_spad + (idx << 2); 413 + writel(val, mmio + AMD_SPAD_OFFSET + offset); 414 + 415 + return 0; 416 + } 417 + 418 + static int amd_ntb_peer_spad_addr(struct ntb_dev *ntb, int idx, 419 + phys_addr_t *spad_addr) 420 + { 421 + struct amd_ntb_dev *ndev = ntb_ndev(ntb); 422 + 423 + if (idx < 0 || idx >= ndev->spad_count) 424 + return -EINVAL; 425 + 426 + if (spad_addr) 427 + *spad_addr = (phys_addr_t)(ndev->self_mmio + AMD_SPAD_OFFSET + 428 + ndev->peer_spad + (idx << 2)); 429 + return 0; 430 + } 431 + 432 + static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int idx) 433 + { 434 + struct amd_ntb_dev *ndev = ntb_ndev(ntb); 435 + void __iomem *mmio = ndev->self_mmio; 436 + u32 offset; 437 + 438 + if (idx < 0 || idx >= ndev->spad_count) 439 + return -EINVAL; 440 + 441 + offset = ndev->peer_spad + (idx << 2); 442 + return readl(mmio + AMD_SPAD_OFFSET + offset); 443 + } 444 + 445 + static int amd_ntb_peer_spad_write(struct ntb_dev *ntb, 446 + int idx, u32 val) 447 + { 448 + struct amd_ntb_dev *ndev = ntb_ndev(ntb); 449 + void __iomem *mmio = ndev->self_mmio; 450 + u32 offset; 451 + 452 + if (idx < 0 || idx >= ndev->spad_count) 453 + return -EINVAL; 454 + 455 + offset = ndev->peer_spad + (idx << 2); 456 + writel(val, mmio + AMD_SPAD_OFFSET + offset); 457 + 458 + return 0; 459 + } 460 + 461 + static const struct ntb_dev_ops amd_ntb_ops = { 462 + .mw_count = amd_ntb_mw_count, 463 + .mw_get_range = amd_ntb_mw_get_range, 464 + .mw_set_trans = amd_ntb_mw_set_trans, 465 + .link_is_up = amd_ntb_link_is_up, 466 + .link_enable = amd_ntb_link_enable, 467 + .link_disable = amd_ntb_link_disable, 468 + .db_valid_mask = amd_ntb_db_valid_mask, 469 + .db_vector_count = amd_ntb_db_vector_count, 470 + .db_vector_mask = amd_ntb_db_vector_mask, 471 + .db_read = amd_ntb_db_read, 472 + .db_clear = amd_ntb_db_clear, 473 + .db_set_mask = amd_ntb_db_set_mask, 474 + .db_clear_mask = amd_ntb_db_clear_mask, 475 + .peer_db_addr = amd_ntb_peer_db_addr, 476 + .peer_db_set = amd_ntb_peer_db_set, 477 + .spad_count = amd_ntb_spad_count, 478 + .spad_read = amd_ntb_spad_read, 479 + .spad_write = amd_ntb_spad_write, 480 + .peer_spad_addr = amd_ntb_peer_spad_addr, 481 + .peer_spad_read = amd_ntb_peer_spad_read, 482 + .peer_spad_write = amd_ntb_peer_spad_write, 483 + }; 484 + 485 + static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit) 486 + { 487 + void __iomem *mmio = ndev->self_mmio; 488 + int reg; 489 + 490 + reg = readl(mmio + AMD_SMUACK_OFFSET); 491 + reg |= bit; 492 + writel(reg, mmio + AMD_SMUACK_OFFSET); 493 + 494 + ndev->peer_sta |= bit; 495 + } 496 + 497 + static void amd_handle_event(struct amd_ntb_dev *ndev, int vec) 498 + { 499 + void __iomem *mmio = ndev->self_mmio; 500 + u32 status; 501 + 502 + status = readl(mmio + AMD_INTSTAT_OFFSET); 503 + if (!(status & AMD_EVENT_INTMASK)) 504 + return; 505 + 506 + dev_dbg(ndev_dev(ndev), "status = 0x%x and vec = %d\n", status, vec); 507 + 508 + status &= AMD_EVENT_INTMASK; 509 + switch (status) { 510 + case AMD_PEER_FLUSH_EVENT: 511 + dev_info(ndev_dev(ndev), "Flush is done.\n"); 512 + break; 513 + case AMD_PEER_RESET_EVENT: 514 + amd_ack_smu(ndev, AMD_PEER_RESET_EVENT); 515 + 516 + /* link down first */ 517 + ntb_link_event(&ndev->ntb); 518 + /* polling peer status */ 519 + schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); 520 + 521 + break; 522 + case AMD_PEER_D3_EVENT: 523 + case AMD_PEER_PMETO_EVENT: 524 + amd_ack_smu(ndev, status); 525 + 526 + /* link down */ 527 + ntb_link_event(&ndev->ntb); 528 + 529 + break; 530 + case AMD_PEER_D0_EVENT: 531 + mmio = ndev->peer_mmio; 532 + status = readl(mmio + AMD_PMESTAT_OFFSET); 533 + /* check if this is WAKEUP event */ 534 + if (status & 0x1) 535 + dev_info(ndev_dev(ndev), "Wakeup is done.\n"); 536 + 537 + amd_ack_smu(ndev, AMD_PEER_D0_EVENT); 538 + 539 + /* start a timer to poll link status */ 540 + schedule_delayed_work(&ndev->hb_timer, 541 + AMD_LINK_HB_TIMEOUT); 542 + break; 543 + default: 544 + dev_info(ndev_dev(ndev), "event status = 0x%x.\n", status); 545 + break; 546 + } 547 + } 548 + 549 + static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec) 550 + { 551 + dev_dbg(ndev_dev(ndev), "vec %d\n", vec); 552 + 553 + if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1)) 554 + amd_handle_event(ndev, vec); 555 + 556 + if (vec < AMD_DB_CNT) 557 + ntb_db_event(&ndev->ntb, vec); 558 + 559 + return IRQ_HANDLED; 560 + } 561 + 562 + static irqreturn_t ndev_vec_isr(int irq, void *dev) 563 + { 564 + struct amd_ntb_vec *nvec = dev; 565 + 566 + return ndev_interrupt(nvec->ndev, nvec->num); 567 + } 568 + 569 + static irqreturn_t ndev_irq_isr(int irq, void *dev) 570 + { 571 + struct amd_ntb_dev *ndev = dev; 572 + 573 + return ndev_interrupt(ndev, irq - ndev_pdev(ndev)->irq); 574 + } 575 + 576 + static int ndev_init_isr(struct amd_ntb_dev *ndev, 577 + int msix_min, int msix_max) 578 + { 579 + struct pci_dev *pdev; 580 + int rc, i, msix_count, node; 581 + 582 + pdev = ndev_pdev(ndev); 583 + 584 + node = dev_to_node(&pdev->dev); 585 + 586 + ndev->db_mask = ndev->db_valid_mask; 587 + 588 + /* Try to set up msix irq */ 589 + ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec), 590 + GFP_KERNEL, node); 591 + if (!ndev->vec) 592 + goto err_msix_vec_alloc; 593 + 594 + ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix), 595 + GFP_KERNEL, node); 596 + if (!ndev->msix) 597 + goto err_msix_alloc; 598 + 599 + for (i = 0; i < msix_max; ++i) 600 + ndev->msix[i].entry = i; 601 + 602 + msix_count = pci_enable_msix_range(pdev, ndev->msix, 603 + msix_min, msix_max); 604 + if (msix_count < 0) 605 + goto err_msix_enable; 606 + 607 + /* NOTE: Disable MSIX if msix count is less than 16 because of 608 + * hardware limitation. 609 + */ 610 + if (msix_count < msix_min) { 611 + pci_disable_msix(pdev); 612 + goto err_msix_enable; 613 + } 614 + 615 + for (i = 0; i < msix_count; ++i) { 616 + ndev->vec[i].ndev = ndev; 617 + ndev->vec[i].num = i; 618 + rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0, 619 + "ndev_vec_isr", &ndev->vec[i]); 620 + if (rc) 621 + goto err_msix_request; 622 + } 623 + 624 + dev_dbg(ndev_dev(ndev), "Using msix interrupts\n"); 625 + ndev->db_count = msix_min; 626 + ndev->msix_vec_count = msix_max; 627 + return 0; 628 + 629 + err_msix_request: 630 + while (i-- > 0) 631 + free_irq(ndev->msix[i].vector, ndev); 632 + pci_disable_msix(pdev); 633 + err_msix_enable: 634 + kfree(ndev->msix); 635 + err_msix_alloc: 636 + kfree(ndev->vec); 637 + err_msix_vec_alloc: 638 + ndev->msix = NULL; 639 + ndev->vec = NULL; 640 + 641 + /* Try to set up msi irq */ 642 + rc = pci_enable_msi(pdev); 643 + if (rc) 644 + goto err_msi_enable; 645 + 646 + rc = request_irq(pdev->irq, ndev_irq_isr, 0, 647 + "ndev_irq_isr", ndev); 648 + if (rc) 649 + goto err_msi_request; 650 + 651 + dev_dbg(ndev_dev(ndev), "Using msi interrupts\n"); 652 + ndev->db_count = 1; 653 + ndev->msix_vec_count = 1; 654 + return 0; 655 + 656 + err_msi_request: 657 + pci_disable_msi(pdev); 658 + err_msi_enable: 659 + 660 + /* Try to set up intx irq */ 661 + pci_intx(pdev, 1); 662 + 663 + rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED, 664 + "ndev_irq_isr", ndev); 665 + if (rc) 666 + goto err_intx_request; 667 + 668 + dev_dbg(ndev_dev(ndev), "Using intx interrupts\n"); 669 + ndev->db_count = 1; 670 + ndev->msix_vec_count = 1; 671 + return 0; 672 + 673 + err_intx_request: 674 + return rc; 675 + } 676 + 677 + static void ndev_deinit_isr(struct amd_ntb_dev *ndev) 678 + { 679 + struct pci_dev *pdev; 680 + void __iomem *mmio = ndev->self_mmio; 681 + int i; 682 + 683 + pdev = ndev_pdev(ndev); 684 + 685 + /* Mask all doorbell interrupts */ 686 + ndev->db_mask = ndev->db_valid_mask; 687 + writel(ndev->db_mask, mmio + AMD_DBMASK_OFFSET); 688 + 689 + if (ndev->msix) { 690 + i = ndev->msix_vec_count; 691 + while (i--) 692 + free_irq(ndev->msix[i].vector, &ndev->vec[i]); 693 + pci_disable_msix(pdev); 694 + kfree(ndev->msix); 695 + kfree(ndev->vec); 696 + } else { 697 + free_irq(pdev->irq, ndev); 698 + if (pci_dev_msi_enabled(pdev)) 699 + pci_disable_msi(pdev); 700 + else 701 + pci_intx(pdev, 0); 702 + } 703 + } 704 + 705 + static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf, 706 + size_t count, loff_t *offp) 707 + { 708 + struct amd_ntb_dev *ndev; 709 + void __iomem *mmio; 710 + char *buf; 711 + size_t buf_size; 712 + ssize_t ret, off; 713 + union { u64 v64; u32 v32; u16 v16; } u; 714 + 715 + ndev = filp->private_data; 716 + mmio = ndev->self_mmio; 717 + 718 + buf_size = min(count, 0x800ul); 719 + 720 + buf = kmalloc(buf_size, GFP_KERNEL); 721 + if (!buf) 722 + return -ENOMEM; 723 + 724 + off = 0; 725 + 726 + off += scnprintf(buf + off, buf_size - off, 727 + "NTB Device Information:\n"); 728 + 729 + off += scnprintf(buf + off, buf_size - off, 730 + "Connection Topology -\t%s\n", 731 + ntb_topo_string(ndev->ntb.topo)); 732 + 733 + off += scnprintf(buf + off, buf_size - off, 734 + "LNK STA -\t\t%#06x\n", ndev->lnk_sta); 735 + 736 + if (!amd_link_is_up(ndev)) { 737 + off += scnprintf(buf + off, buf_size - off, 738 + "Link Status -\t\tDown\n"); 739 + } else { 740 + off += scnprintf(buf + off, buf_size - off, 741 + "Link Status -\t\tUp\n"); 742 + off += scnprintf(buf + off, buf_size - off, 743 + "Link Speed -\t\tPCI-E Gen %u\n", 744 + NTB_LNK_STA_SPEED(ndev->lnk_sta)); 745 + off += scnprintf(buf + off, buf_size - off, 746 + "Link Width -\t\tx%u\n", 747 + NTB_LNK_STA_WIDTH(ndev->lnk_sta)); 748 + } 749 + 750 + off += scnprintf(buf + off, buf_size - off, 751 + "Memory Window Count -\t%u\n", ndev->mw_count); 752 + off += scnprintf(buf + off, buf_size - off, 753 + "Scratchpad Count -\t%u\n", ndev->spad_count); 754 + off += scnprintf(buf + off, buf_size - off, 755 + "Doorbell Count -\t%u\n", ndev->db_count); 756 + off += scnprintf(buf + off, buf_size - off, 757 + "MSIX Vector Count -\t%u\n", ndev->msix_vec_count); 758 + 759 + off += scnprintf(buf + off, buf_size - off, 760 + "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask); 761 + 762 + u.v32 = readl(ndev->self_mmio + AMD_DBMASK_OFFSET); 763 + off += scnprintf(buf + off, buf_size - off, 764 + "Doorbell Mask -\t\t\t%#06x\n", u.v32); 765 + 766 + u.v32 = readl(mmio + AMD_DBSTAT_OFFSET); 767 + off += scnprintf(buf + off, buf_size - off, 768 + "Doorbell Bell -\t\t\t%#06x\n", u.v32); 769 + 770 + off += scnprintf(buf + off, buf_size - off, 771 + "\nNTB Incoming XLAT:\n"); 772 + 773 + u.v64 = read64(mmio + AMD_BAR1XLAT_OFFSET); 774 + off += scnprintf(buf + off, buf_size - off, 775 + "XLAT1 -\t\t%#018llx\n", u.v64); 776 + 777 + u.v64 = read64(ndev->self_mmio + AMD_BAR23XLAT_OFFSET); 778 + off += scnprintf(buf + off, buf_size - off, 779 + "XLAT23 -\t\t%#018llx\n", u.v64); 780 + 781 + u.v64 = read64(ndev->self_mmio + AMD_BAR45XLAT_OFFSET); 782 + off += scnprintf(buf + off, buf_size - off, 783 + "XLAT45 -\t\t%#018llx\n", u.v64); 784 + 785 + u.v32 = readl(mmio + AMD_BAR1LMT_OFFSET); 786 + off += scnprintf(buf + off, buf_size - off, 787 + "LMT1 -\t\t\t%#06x\n", u.v32); 788 + 789 + u.v64 = read64(ndev->self_mmio + AMD_BAR23LMT_OFFSET); 790 + off += scnprintf(buf + off, buf_size - off, 791 + "LMT23 -\t\t\t%#018llx\n", u.v64); 792 + 793 + u.v64 = read64(ndev->self_mmio + AMD_BAR45LMT_OFFSET); 794 + off += scnprintf(buf + off, buf_size - off, 795 + "LMT45 -\t\t\t%#018llx\n", u.v64); 796 + 797 + ret = simple_read_from_buffer(ubuf, count, offp, buf, off); 798 + kfree(buf); 799 + return ret; 800 + } 801 + 802 + static void ndev_init_debugfs(struct amd_ntb_dev *ndev) 803 + { 804 + if (!debugfs_dir) { 805 + ndev->debugfs_dir = NULL; 806 + ndev->debugfs_info = NULL; 807 + } else { 808 + ndev->debugfs_dir = 809 + debugfs_create_dir(ndev_name(ndev), debugfs_dir); 810 + if (!ndev->debugfs_dir) 811 + ndev->debugfs_info = NULL; 812 + else 813 + ndev->debugfs_info = 814 + debugfs_create_file("info", S_IRUSR, 815 + ndev->debugfs_dir, ndev, 816 + &amd_ntb_debugfs_info); 817 + } 818 + } 819 + 820 + static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev) 821 + { 822 + debugfs_remove_recursive(ndev->debugfs_dir); 823 + } 824 + 825 + static inline void ndev_init_struct(struct amd_ntb_dev *ndev, 826 + struct pci_dev *pdev) 827 + { 828 + ndev->ntb.pdev = pdev; 829 + ndev->ntb.topo = NTB_TOPO_NONE; 830 + ndev->ntb.ops = &amd_ntb_ops; 831 + ndev->int_mask = AMD_EVENT_INTMASK; 832 + spin_lock_init(&ndev->db_mask_lock); 833 + } 834 + 835 + static int amd_poll_link(struct amd_ntb_dev *ndev) 836 + { 837 + void __iomem *mmio = ndev->peer_mmio; 838 + u32 reg, stat; 839 + int rc; 840 + 841 + reg = readl(mmio + AMD_SIDEINFO_OFFSET); 842 + reg &= NTB_LIN_STA_ACTIVE_BIT; 843 + 844 + dev_dbg(ndev_dev(ndev), "%s: reg_val = 0x%x.\n", __func__, reg); 845 + 846 + if (reg == ndev->cntl_sta) 847 + return 0; 848 + 849 + ndev->cntl_sta = reg; 850 + 851 + rc = pci_read_config_dword(ndev->ntb.pdev, 852 + AMD_LINK_STATUS_OFFSET, &stat); 853 + if (rc) 854 + return 0; 855 + ndev->lnk_sta = stat; 856 + 857 + return 1; 858 + } 859 + 860 + static void amd_link_hb(struct work_struct *work) 861 + { 862 + struct amd_ntb_dev *ndev = hb_ndev(work); 863 + 864 + if (amd_poll_link(ndev)) 865 + ntb_link_event(&ndev->ntb); 866 + 867 + if (!amd_link_is_up(ndev)) 868 + schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); 869 + } 870 + 871 + static int amd_init_isr(struct amd_ntb_dev *ndev) 872 + { 873 + return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT); 874 + } 875 + 876 + static void amd_init_side_info(struct amd_ntb_dev *ndev) 877 + { 878 + void __iomem *mmio = ndev->self_mmio; 879 + unsigned int reg; 880 + 881 + reg = readl(mmio + AMD_SIDEINFO_OFFSET); 882 + if (!(reg & AMD_SIDE_READY)) { 883 + reg |= AMD_SIDE_READY; 884 + writel(reg, mmio + AMD_SIDEINFO_OFFSET); 885 + } 886 + } 887 + 888 + static void amd_deinit_side_info(struct amd_ntb_dev *ndev) 889 + { 890 + void __iomem *mmio = ndev->self_mmio; 891 + unsigned int reg; 892 + 893 + reg = readl(mmio + AMD_SIDEINFO_OFFSET); 894 + if (reg & AMD_SIDE_READY) { 895 + reg &= ~AMD_SIDE_READY; 896 + writel(reg, mmio + AMD_SIDEINFO_OFFSET); 897 + readl(mmio + AMD_SIDEINFO_OFFSET); 898 + } 899 + } 900 + 901 + static int amd_init_ntb(struct amd_ntb_dev *ndev) 902 + { 903 + void __iomem *mmio = ndev->self_mmio; 904 + 905 + ndev->mw_count = AMD_MW_CNT; 906 + ndev->spad_count = AMD_SPADS_CNT; 907 + ndev->db_count = AMD_DB_CNT; 908 + 909 + switch (ndev->ntb.topo) { 910 + case NTB_TOPO_PRI: 911 + case NTB_TOPO_SEC: 912 + ndev->spad_count >>= 1; 913 + if (ndev->ntb.topo == NTB_TOPO_PRI) { 914 + ndev->self_spad = 0; 915 + ndev->peer_spad = 0x20; 916 + } else { 917 + ndev->self_spad = 0x20; 918 + ndev->peer_spad = 0; 919 + } 920 + 921 + INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb); 922 + schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT); 923 + 924 + break; 925 + default: 926 + dev_err(ndev_dev(ndev), "AMD NTB does not support B2B mode.\n"); 927 + return -EINVAL; 928 + } 929 + 930 + ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; 931 + 932 + /* Mask event interrupts */ 933 + writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET); 934 + 935 + return 0; 936 + } 937 + 938 + static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev) 939 + { 940 + void __iomem *mmio = ndev->self_mmio; 941 + u32 info; 942 + 943 + info = readl(mmio + AMD_SIDEINFO_OFFSET); 944 + if (info & AMD_SIDE_MASK) 945 + return NTB_TOPO_SEC; 946 + else 947 + return NTB_TOPO_PRI; 948 + } 949 + 950 + static int amd_init_dev(struct amd_ntb_dev *ndev) 951 + { 952 + struct pci_dev *pdev; 953 + int rc = 0; 954 + 955 + pdev = ndev_pdev(ndev); 956 + 957 + ndev->ntb.topo = amd_get_topo(ndev); 958 + dev_dbg(ndev_dev(ndev), "AMD NTB topo is %s\n", 959 + ntb_topo_string(ndev->ntb.topo)); 960 + 961 + rc = amd_init_ntb(ndev); 962 + if (rc) 963 + return rc; 964 + 965 + rc = amd_init_isr(ndev); 966 + if (rc) { 967 + dev_err(ndev_dev(ndev), "fail to init isr.\n"); 968 + return rc; 969 + } 970 + 971 + ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; 972 + 973 + return 0; 974 + } 975 + 976 + static void amd_deinit_dev(struct amd_ntb_dev *ndev) 977 + { 978 + cancel_delayed_work_sync(&ndev->hb_timer); 979 + 980 + ndev_deinit_isr(ndev); 981 + } 982 + 983 + static int amd_ntb_init_pci(struct amd_ntb_dev *ndev, 984 + struct pci_dev *pdev) 985 + { 986 + int rc; 987 + 988 + pci_set_drvdata(pdev, ndev); 989 + 990 + rc = pci_enable_device(pdev); 991 + if (rc) 992 + goto err_pci_enable; 993 + 994 + rc = pci_request_regions(pdev, NTB_NAME); 995 + if (rc) 996 + goto err_pci_regions; 997 + 998 + pci_set_master(pdev); 999 + 1000 + rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 1001 + if (rc) { 1002 + rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 1003 + if (rc) 1004 + goto err_dma_mask; 1005 + dev_warn(ndev_dev(ndev), "Cannot DMA highmem\n"); 1006 + } 1007 + 1008 + rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 1009 + if (rc) { 1010 + rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 1011 + if (rc) 1012 + goto err_dma_mask; 1013 + dev_warn(ndev_dev(ndev), "Cannot DMA consistent highmem\n"); 1014 + } 1015 + 1016 + ndev->self_mmio = pci_iomap(pdev, 0, 0); 1017 + if (!ndev->self_mmio) { 1018 + rc = -EIO; 1019 + goto err_dma_mask; 1020 + } 1021 + ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET; 1022 + 1023 + return 0; 1024 + 1025 + err_dma_mask: 1026 + pci_clear_master(pdev); 1027 + err_pci_regions: 1028 + pci_disable_device(pdev); 1029 + err_pci_enable: 1030 + pci_set_drvdata(pdev, NULL); 1031 + return rc; 1032 + } 1033 + 1034 + static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev) 1035 + { 1036 + struct pci_dev *pdev = ndev_pdev(ndev); 1037 + 1038 + pci_iounmap(pdev, ndev->self_mmio); 1039 + 1040 + pci_clear_master(pdev); 1041 + pci_release_regions(pdev); 1042 + pci_disable_device(pdev); 1043 + pci_set_drvdata(pdev, NULL); 1044 + } 1045 + 1046 + static int amd_ntb_pci_probe(struct pci_dev *pdev, 1047 + const struct pci_device_id *id) 1048 + { 1049 + struct amd_ntb_dev *ndev; 1050 + int rc, node; 1051 + 1052 + node = dev_to_node(&pdev->dev); 1053 + 1054 + ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node); 1055 + if (!ndev) { 1056 + rc = -ENOMEM; 1057 + goto err_ndev; 1058 + } 1059 + 1060 + ndev_init_struct(ndev, pdev); 1061 + 1062 + rc = amd_ntb_init_pci(ndev, pdev); 1063 + if (rc) 1064 + goto err_init_pci; 1065 + 1066 + rc = amd_init_dev(ndev); 1067 + if (rc) 1068 + goto err_init_dev; 1069 + 1070 + /* write side info */ 1071 + amd_init_side_info(ndev); 1072 + 1073 + amd_poll_link(ndev); 1074 + 1075 + ndev_init_debugfs(ndev); 1076 + 1077 + rc = ntb_register_device(&ndev->ntb); 1078 + if (rc) 1079 + goto err_register; 1080 + 1081 + dev_info(&pdev->dev, "NTB device registered.\n"); 1082 + 1083 + return 0; 1084 + 1085 + err_register: 1086 + ndev_deinit_debugfs(ndev); 1087 + amd_deinit_dev(ndev); 1088 + err_init_dev: 1089 + amd_ntb_deinit_pci(ndev); 1090 + err_init_pci: 1091 + kfree(ndev); 1092 + err_ndev: 1093 + return rc; 1094 + } 1095 + 1096 + static void amd_ntb_pci_remove(struct pci_dev *pdev) 1097 + { 1098 + struct amd_ntb_dev *ndev = pci_get_drvdata(pdev); 1099 + 1100 + ntb_unregister_device(&ndev->ntb); 1101 + ndev_deinit_debugfs(ndev); 1102 + amd_deinit_side_info(ndev); 1103 + amd_deinit_dev(ndev); 1104 + amd_ntb_deinit_pci(ndev); 1105 + kfree(ndev); 1106 + } 1107 + 1108 + static const struct file_operations amd_ntb_debugfs_info = { 1109 + .owner = THIS_MODULE, 1110 + .open = simple_open, 1111 + .read = ndev_debugfs_read, 1112 + }; 1113 + 1114 + static const struct pci_device_id amd_ntb_pci_tbl[] = { 1115 + {PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_NTB)}, 1116 + {0} 1117 + }; 1118 + MODULE_DEVICE_TABLE(pci, amd_ntb_pci_tbl); 1119 + 1120 + static struct pci_driver amd_ntb_pci_driver = { 1121 + .name = KBUILD_MODNAME, 1122 + .id_table = amd_ntb_pci_tbl, 1123 + .probe = amd_ntb_pci_probe, 1124 + .remove = amd_ntb_pci_remove, 1125 + }; 1126 + 1127 + static int __init amd_ntb_pci_driver_init(void) 1128 + { 1129 + pr_info("%s %s\n", NTB_DESC, NTB_VER); 1130 + 1131 + if (debugfs_initialized()) 1132 + debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 1133 + 1134 + return pci_register_driver(&amd_ntb_pci_driver); 1135 + } 1136 + module_init(amd_ntb_pci_driver_init); 1137 + 1138 + static void __exit amd_ntb_pci_driver_exit(void) 1139 + { 1140 + pci_unregister_driver(&amd_ntb_pci_driver); 1141 + debugfs_remove_recursive(debugfs_dir); 1142 + } 1143 + module_exit(amd_ntb_pci_driver_exit);

+217

drivers/ntb/hw/amd/ntb_hw_amd.h

··· 1 + /* 2 + * This file is provided under a dual BSD/GPLv2 license. When using or 3 + * redistributing this file, you may do so under either license. 4 + * 5 + * GPL LICENSE SUMMARY 6 + * 7 + * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved. 8 + * 9 + * This program is free software; you can redistribute it and/or modify 10 + * it under the terms of version 2 of the GNU General Public License as 11 + * published by the Free Software Foundation. 12 + * 13 + * BSD LICENSE 14 + * 15 + * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved. 16 + * 17 + * Redistribution and use in source and binary forms, with or without 18 + * modification, are permitted provided that the following conditions 19 + * are met: 20 + * 21 + * * Redistributions of source code must retain the above copyright 22 + * notice, this list of conditions and the following disclaimer. 23 + * * Redistributions in binary form must reproduce the above copy 24 + * notice, this list of conditions and the following disclaimer in 25 + * the documentation and/or other materials provided with the 26 + * distribution. 27 + * * Neither the name of AMD Corporation nor the names of its 28 + * contributors may be used to endorse or promote products derived 29 + * from this software without specific prior written permission. 30 + * 31 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 32 + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 33 + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 34 + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 35 + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 36 + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 37 + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 38 + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 39 + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 40 + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 41 + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 42 + * 43 + * AMD PCIe NTB Linux driver 44 + * 45 + * Contact Information: 46 + * Xiangliang Yu <Xiangliang.Yu@amd.com> 47 + */ 48 + 49 + #ifndef NTB_HW_AMD_H 50 + #define NTB_HW_AMD_H 51 + 52 + #include <linux/ntb.h> 53 + #include <linux/pci.h> 54 + 55 + #define PCI_DEVICE_ID_AMD_NTB 0x145B 56 + #define AMD_LINK_HB_TIMEOUT msecs_to_jiffies(1000) 57 + #define AMD_LINK_STATUS_OFFSET 0x68 58 + #define NTB_LIN_STA_ACTIVE_BIT 0x00000002 59 + #define NTB_LNK_STA_SPEED_MASK 0x000F0000 60 + #define NTB_LNK_STA_WIDTH_MASK 0x03F00000 61 + #define NTB_LNK_STA_ACTIVE(x) (!!((x) & NTB_LIN_STA_ACTIVE_BIT)) 62 + #define NTB_LNK_STA_SPEED(x) (((x) & NTB_LNK_STA_SPEED_MASK) >> 16) 63 + #define NTB_LNK_STA_WIDTH(x) (((x) & NTB_LNK_STA_WIDTH_MASK) >> 20) 64 + 65 + #ifndef read64 66 + #ifdef readq 67 + #define read64 readq 68 + #else 69 + #define read64 _read64 70 + static inline u64 _read64(void __iomem *mmio) 71 + { 72 + u64 low, high; 73 + 74 + low = readl(mmio); 75 + high = readl(mmio + sizeof(u32)); 76 + return low | (high << 32); 77 + } 78 + #endif 79 + #endif 80 + 81 + #ifndef write64 82 + #ifdef writeq 83 + #define write64 writeq 84 + #else 85 + #define write64 _write64 86 + static inline void _write64(u64 val, void __iomem *mmio) 87 + { 88 + writel(val, mmio); 89 + writel(val >> 32, mmio + sizeof(u32)); 90 + } 91 + #endif 92 + #endif 93 + 94 + enum { 95 + /* AMD NTB Capability */ 96 + AMD_MW_CNT = 3, 97 + AMD_DB_CNT = 16, 98 + AMD_MSIX_VECTOR_CNT = 24, 99 + AMD_SPADS_CNT = 16, 100 + 101 + /* AMD NTB register offset */ 102 + AMD_CNTL_OFFSET = 0x200, 103 + 104 + /* NTB control register bits */ 105 + PMM_REG_CTL = BIT(21), 106 + SMM_REG_CTL = BIT(20), 107 + SMM_REG_ACC_PATH = BIT(18), 108 + PMM_REG_ACC_PATH = BIT(17), 109 + NTB_CLK_EN = BIT(16), 110 + 111 + AMD_STA_OFFSET = 0x204, 112 + AMD_PGSLV_OFFSET = 0x208, 113 + AMD_SPAD_MUX_OFFSET = 0x20C, 114 + AMD_SPAD_OFFSET = 0x210, 115 + AMD_RSMU_HCID = 0x250, 116 + AMD_RSMU_SIID = 0x254, 117 + AMD_PSION_OFFSET = 0x300, 118 + AMD_SSION_OFFSET = 0x330, 119 + AMD_MMINDEX_OFFSET = 0x400, 120 + AMD_MMDATA_OFFSET = 0x404, 121 + AMD_SIDEINFO_OFFSET = 0x408, 122 + 123 + AMD_SIDE_MASK = BIT(0), 124 + AMD_SIDE_READY = BIT(1), 125 + 126 + /* limit register */ 127 + AMD_ROMBARLMT_OFFSET = 0x410, 128 + AMD_BAR1LMT_OFFSET = 0x414, 129 + AMD_BAR23LMT_OFFSET = 0x418, 130 + AMD_BAR45LMT_OFFSET = 0x420, 131 + /* xlat address */ 132 + AMD_POMBARXLAT_OFFSET = 0x428, 133 + AMD_BAR1XLAT_OFFSET = 0x430, 134 + AMD_BAR23XLAT_OFFSET = 0x438, 135 + AMD_BAR45XLAT_OFFSET = 0x440, 136 + /* doorbell and interrupt */ 137 + AMD_DBFM_OFFSET = 0x450, 138 + AMD_DBREQ_OFFSET = 0x454, 139 + AMD_MIRRDBSTAT_OFFSET = 0x458, 140 + AMD_DBMASK_OFFSET = 0x45C, 141 + AMD_DBSTAT_OFFSET = 0x460, 142 + AMD_INTMASK_OFFSET = 0x470, 143 + AMD_INTSTAT_OFFSET = 0x474, 144 + 145 + /* event type */ 146 + AMD_PEER_FLUSH_EVENT = BIT(0), 147 + AMD_PEER_RESET_EVENT = BIT(1), 148 + AMD_PEER_D3_EVENT = BIT(2), 149 + AMD_PEER_PMETO_EVENT = BIT(3), 150 + AMD_PEER_D0_EVENT = BIT(4), 151 + AMD_EVENT_INTMASK = (AMD_PEER_FLUSH_EVENT | 152 + AMD_PEER_RESET_EVENT | AMD_PEER_D3_EVENT | 153 + AMD_PEER_PMETO_EVENT | AMD_PEER_D0_EVENT), 154 + 155 + AMD_PMESTAT_OFFSET = 0x480, 156 + AMD_PMSGTRIG_OFFSET = 0x490, 157 + AMD_LTRLATENCY_OFFSET = 0x494, 158 + AMD_FLUSHTRIG_OFFSET = 0x498, 159 + 160 + /* SMU register*/ 161 + AMD_SMUACK_OFFSET = 0x4A0, 162 + AMD_SINRST_OFFSET = 0x4A4, 163 + AMD_RSPNUM_OFFSET = 0x4A8, 164 + AMD_SMU_SPADMUTEX = 0x4B0, 165 + AMD_SMU_SPADOFFSET = 0x4B4, 166 + 167 + AMD_PEER_OFFSET = 0x400, 168 + }; 169 + 170 + struct amd_ntb_dev; 171 + 172 + struct amd_ntb_vec { 173 + struct amd_ntb_dev *ndev; 174 + int num; 175 + }; 176 + 177 + struct amd_ntb_dev { 178 + struct ntb_dev ntb; 179 + 180 + u32 ntb_side; 181 + u32 lnk_sta; 182 + u32 cntl_sta; 183 + u32 peer_sta; 184 + 185 + unsigned char mw_count; 186 + unsigned char spad_count; 187 + unsigned char db_count; 188 + unsigned char msix_vec_count; 189 + 190 + u64 db_valid_mask; 191 + u64 db_mask; 192 + u32 int_mask; 193 + 194 + struct msix_entry *msix; 195 + struct amd_ntb_vec *vec; 196 + 197 + /* synchronize rmw access of db_mask and hw reg */ 198 + spinlock_t db_mask_lock; 199 + 200 + void __iomem *self_mmio; 201 + void __iomem *peer_mmio; 202 + unsigned int self_spad; 203 + unsigned int peer_spad; 204 + 205 + struct delayed_work hb_timer; 206 + 207 + struct dentry *debugfs_dir; 208 + struct dentry *debugfs_info; 209 + }; 210 + 211 + #define ndev_pdev(ndev) ((ndev)->ntb.pdev) 212 + #define ndev_name(ndev) pci_name(ndev_pdev(ndev)) 213 + #define ndev_dev(ndev) (&ndev_pdev(ndev)->dev) 214 + #define ntb_ndev(__ntb) container_of(__ntb, struct amd_ntb_dev, ntb) 215 + #define hb_ndev(__work) container_of(__work, struct amd_ntb_dev, hb_timer.work) 216 + 217 + #endif

+2 -2

drivers/ntb/hw/intel/ntb_hw_intel.c

··· 875 875 limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar); 876 876 877 877 if (bar < 4 || !ndev->bar4_split) { 878 - base = ioread64(mmio + base_reg); 878 + base = ioread64(mmio + base_reg) & NTB_BAR_MASK_64; 879 879 880 880 /* Set the limit if supported, if size is not mw_size */ 881 881 if (limit_reg && size != mw_size) ··· 906 906 if ((addr + size) & (~0ull << 32)) 907 907 return -EINVAL; 908 908 909 - base = ioread32(mmio + base_reg); 909 + base = ioread32(mmio + base_reg) & NTB_BAR_MASK_32; 910 910 911 911 /* Set the limit if supported, if size is not mw_size */ 912 912 if (limit_reg && size != mw_size)

+6 -2

drivers/ntb/hw/intel/ntb_hw_intel.h

··· 245 245 #define NTB_UNSAFE_DB BIT_ULL(0) 246 246 #define NTB_UNSAFE_SPAD BIT_ULL(1) 247 247 248 + #define NTB_BAR_MASK_64 ~(0xfull) 249 + #define NTB_BAR_MASK_32 ~(0xfu) 250 + 248 251 struct intel_ntb_dev; 249 252 250 253 struct intel_ntb_reg { ··· 337 334 #define ndev_pdev(ndev) ((ndev)->ntb.pdev) 338 335 #define ndev_name(ndev) pci_name(ndev_pdev(ndev)) 339 336 #define ndev_dev(ndev) (&ndev_pdev(ndev)->dev) 340 - #define ntb_ndev(ntb) container_of(ntb, struct intel_ntb_dev, ntb) 341 - #define hb_ndev(work) container_of(work, struct intel_ntb_dev, hb_timer.work) 337 + #define ntb_ndev(__ntb) container_of(__ntb, struct intel_ntb_dev, ntb) 338 + #define hb_ndev(__work) container_of(__work, struct intel_ntb_dev, \ 339 + hb_timer.work) 342 340 343 341 #endif

+42 -8

drivers/ntb/ntb_transport.c

··· 171 171 u64 rx_err_ver; 172 172 u64 rx_memcpy; 173 173 u64 rx_async; 174 + u64 dma_rx_prep_err; 174 175 u64 tx_bytes; 175 176 u64 tx_pkts; 176 177 u64 tx_ring_full; 177 178 u64 tx_err_no_buf; 178 179 u64 tx_memcpy; 179 180 u64 tx_async; 181 + u64 dma_tx_prep_err; 180 182 }; 181 183 182 184 struct ntb_transport_mw { ··· 251 249 #define QP_TO_MW(nt, qp) ((qp) % nt->mw_count) 252 250 #define NTB_QP_DEF_NUM_ENTRIES 100 253 251 #define NTB_LINK_DOWN_TIMEOUT 10 252 + #define DMA_RETRIES 20 253 + #define DMA_OUT_RESOURCE_TO 50 254 254 255 255 static void ntb_transport_rxc_db(unsigned long data); 256 256 static const struct ntb_ctx_ops ntb_transport_ops; ··· 505 501 out_offset += snprintf(buf + out_offset, out_count - out_offset, 506 502 "free tx - \t%u\n", 507 503 ntb_transport_tx_free_entry(qp)); 504 + out_offset += snprintf(buf + out_offset, out_count - out_offset, 505 + "DMA tx prep err - \t%llu\n", 506 + qp->dma_tx_prep_err); 507 + out_offset += snprintf(buf + out_offset, out_count - out_offset, 508 + "DMA rx prep err - \t%llu\n", 509 + qp->dma_rx_prep_err); 508 510 509 511 out_offset += snprintf(buf + out_offset, out_count - out_offset, 510 512 "\n"); ··· 736 726 qp->tx_err_no_buf = 0; 737 727 qp->tx_memcpy = 0; 738 728 qp->tx_async = 0; 729 + qp->dma_tx_prep_err = 0; 730 + qp->dma_rx_prep_err = 0; 739 731 } 740 732 741 733 static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp) ··· 1240 1228 struct dmaengine_unmap_data *unmap; 1241 1229 dma_cookie_t cookie; 1242 1230 void *buf = entry->buf; 1231 + int retries = 0; 1243 1232 1244 1233 len = entry->len; 1245 1234 ··· 1276 1263 1277 1264 unmap->from_cnt = 1; 1278 1265 1279 - txd = device->device_prep_dma_memcpy(chan, unmap->addr[1], 1280 - unmap->addr[0], len, 1281 - DMA_PREP_INTERRUPT); 1282 - if (!txd) 1266 + for (retries = 0; retries < DMA_RETRIES; retries++) { 1267 + txd = device->device_prep_dma_memcpy(chan, unmap->addr[1], 1268 + unmap->addr[0], len, 1269 + DMA_PREP_INTERRUPT); 1270 + if (txd) 1271 + break; 1272 + 1273 + set_current_state(TASK_INTERRUPTIBLE); 1274 + schedule_timeout(DMA_OUT_RESOURCE_TO); 1275 + } 1276 + 1277 + if (!txd) { 1278 + qp->dma_rx_prep_err++; 1283 1279 goto err_get_unmap; 1280 + } 1284 1281 1285 1282 txd->callback = ntb_rx_copy_callback; 1286 1283 txd->callback_param = entry; ··· 1483 1460 void __iomem *offset; 1484 1461 size_t len = entry->len; 1485 1462 void *buf = entry->buf; 1463 + int retries = 0; 1486 1464 1487 1465 offset = qp->tx_mw + qp->tx_max_frame * qp->tx_index; 1488 1466 hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header); ··· 1518 1494 1519 1495 unmap->to_cnt = 1; 1520 1496 1521 - txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0], len, 1522 - DMA_PREP_INTERRUPT); 1523 - if (!txd) 1497 + for (retries = 0; retries < DMA_RETRIES; retries++) { 1498 + txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0], 1499 + len, DMA_PREP_INTERRUPT); 1500 + if (txd) 1501 + break; 1502 + 1503 + set_current_state(TASK_INTERRUPTIBLE); 1504 + schedule_timeout(DMA_OUT_RESOURCE_TO); 1505 + } 1506 + 1507 + if (!txd) { 1508 + qp->dma_tx_prep_err++; 1524 1509 goto err_get_unmap; 1510 + } 1525 1511 1526 1512 txd->callback = ntb_tx_copy_callback; 1527 1513 txd->callback_param = entry; ··· 1566 1532 1567 1533 if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { 1568 1534 if (qp->tx_handler) 1569 - qp->tx_handler(qp->cb_data, qp, NULL, -EIO); 1535 + qp->tx_handler(qp, qp->cb_data, NULL, -EIO); 1570 1536 1571 1537 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 1572 1538 &qp->tx_free_q);

+8

drivers/ntb/test/Kconfig

··· 17 17 functioning at a basic level. 18 18 19 19 If unsure, say N. 20 + 21 + config NTB_PERF 22 + tristate "NTB RAW Perf Measuring Tool" 23 + help 24 + This is a tool to measure raw NTB performance by transferring data 25 + to and from the window without additional software interaction. 26 + 27 + If unsure, say N.

+1

drivers/ntb/test/Makefile

··· 1 1 obj-$(CONFIG_NTB_PINGPONG) += ntb_pingpong.o 2 2 obj-$(CONFIG_NTB_TOOL) += ntb_tool.o 3 + obj-$(CONFIG_NTB_PERF) += ntb_perf.o

+748

drivers/ntb/test/ntb_perf.c

··· 1 + /* 2 + * This file is provided under a dual BSD/GPLv2 license. When using or 3 + * redistributing this file, you may do so under either license. 4 + * 5 + * GPL LICENSE SUMMARY 6 + * 7 + * Copyright(c) 2015 Intel Corporation. All rights reserved. 8 + * 9 + * This program is free software; you can redistribute it and/or modify 10 + * it under the terms of version 2 of the GNU General Public License as 11 + * published by the Free Software Foundation. 12 + * 13 + * BSD LICENSE 14 + * 15 + * Copyright(c) 2015 Intel Corporation. All rights reserved. 16 + * 17 + * Redistribution and use in source and binary forms, with or without 18 + * modification, are permitted provided that the following conditions 19 + * are met: 20 + * 21 + * * Redistributions of source code must retain the above copyright 22 + * notice, this list of conditions and the following disclaimer. 23 + * * Redistributions in binary form must reproduce the above copy 24 + * notice, this list of conditions and the following disclaimer in 25 + * the documentation and/or other materials provided with the 26 + * distribution. 27 + * * Neither the name of Intel Corporation nor the names of its 28 + * contributors may be used to endorse or promote products derived 29 + * from this software without specific prior written permission. 30 + * 31 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 32 + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 33 + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 34 + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 35 + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 36 + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 37 + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 38 + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 39 + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 40 + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 41 + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 42 + * 43 + * PCIe NTB Perf Linux driver 44 + */ 45 + 46 + #include <linux/init.h> 47 + #include <linux/kernel.h> 48 + #include <linux/module.h> 49 + #include <linux/kthread.h> 50 + #include <linux/time.h> 51 + #include <linux/timer.h> 52 + #include <linux/dma-mapping.h> 53 + #include <linux/pci.h> 54 + #include <linux/slab.h> 55 + #include <linux/spinlock.h> 56 + #include <linux/debugfs.h> 57 + #include <linux/dmaengine.h> 58 + #include <linux/delay.h> 59 + #include <linux/sizes.h> 60 + #include <linux/ntb.h> 61 + 62 + #define DRIVER_NAME "ntb_perf" 63 + #define DRIVER_DESCRIPTION "PCIe NTB Performance Measurement Tool" 64 + 65 + #define DRIVER_LICENSE "Dual BSD/GPL" 66 + #define DRIVER_VERSION "1.0" 67 + #define DRIVER_AUTHOR "Dave Jiang <dave.jiang@intel.com>" 68 + 69 + #define PERF_LINK_DOWN_TIMEOUT 10 70 + #define PERF_VERSION 0xffff0001 71 + #define MAX_THREADS 32 72 + #define MAX_TEST_SIZE SZ_1M 73 + #define MAX_SRCS 32 74 + #define DMA_OUT_RESOURCE_TO 50 75 + #define DMA_RETRIES 20 76 + #define SZ_4G (1ULL << 32) 77 + #define MAX_SEG_ORDER 20 /* no larger than 1M for kmalloc buffer */ 78 + 79 + MODULE_LICENSE(DRIVER_LICENSE); 80 + MODULE_VERSION(DRIVER_VERSION); 81 + MODULE_AUTHOR(DRIVER_AUTHOR); 82 + MODULE_DESCRIPTION(DRIVER_DESCRIPTION); 83 + 84 + static struct dentry *perf_debugfs_dir; 85 + 86 + static unsigned int seg_order = 19; /* 512K */ 87 + module_param(seg_order, uint, 0644); 88 + MODULE_PARM_DESC(seg_order, "size order [n^2] of buffer segment for testing"); 89 + 90 + static unsigned int run_order = 32; /* 4G */ 91 + module_param(run_order, uint, 0644); 92 + MODULE_PARM_DESC(run_order, "size order [n^2] of total data to transfer"); 93 + 94 + static bool use_dma; /* default to 0 */ 95 + module_param(use_dma, bool, 0644); 96 + MODULE_PARM_DESC(use_dma, "Using DMA engine to measure performance"); 97 + 98 + struct perf_mw { 99 + phys_addr_t phys_addr; 100 + resource_size_t phys_size; 101 + resource_size_t xlat_align; 102 + resource_size_t xlat_align_size; 103 + void __iomem *vbase; 104 + size_t xlat_size; 105 + size_t buf_size; 106 + void *virt_addr; 107 + dma_addr_t dma_addr; 108 + }; 109 + 110 + struct perf_ctx; 111 + 112 + struct pthr_ctx { 113 + struct task_struct *thread; 114 + struct perf_ctx *perf; 115 + atomic_t dma_sync; 116 + struct dma_chan *dma_chan; 117 + int dma_prep_err; 118 + int src_idx; 119 + void *srcs[MAX_SRCS]; 120 + }; 121 + 122 + struct perf_ctx { 123 + struct ntb_dev *ntb; 124 + spinlock_t db_lock; 125 + struct perf_mw mw; 126 + bool link_is_up; 127 + struct work_struct link_cleanup; 128 + struct delayed_work link_work; 129 + struct dentry *debugfs_node_dir; 130 + struct dentry *debugfs_run; 131 + struct dentry *debugfs_threads; 132 + u8 perf_threads; 133 + bool run; 134 + struct pthr_ctx pthr_ctx[MAX_THREADS]; 135 + atomic_t tsync; 136 + }; 137 + 138 + enum { 139 + VERSION = 0, 140 + MW_SZ_HIGH, 141 + MW_SZ_LOW, 142 + SPAD_MSG, 143 + SPAD_ACK, 144 + MAX_SPAD 145 + }; 146 + 147 + static void perf_link_event(void *ctx) 148 + { 149 + struct perf_ctx *perf = ctx; 150 + 151 + if (ntb_link_is_up(perf->ntb, NULL, NULL) == 1) 152 + schedule_delayed_work(&perf->link_work, 2*HZ); 153 + else 154 + schedule_work(&perf->link_cleanup); 155 + } 156 + 157 + static void perf_db_event(void *ctx, int vec) 158 + { 159 + struct perf_ctx *perf = ctx; 160 + u64 db_bits, db_mask; 161 + 162 + db_mask = ntb_db_vector_mask(perf->ntb, vec); 163 + db_bits = ntb_db_read(perf->ntb); 164 + 165 + dev_dbg(&perf->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n", 166 + vec, db_mask, db_bits); 167 + } 168 + 169 + static const struct ntb_ctx_ops perf_ops = { 170 + .link_event = perf_link_event, 171 + .db_event = perf_db_event, 172 + }; 173 + 174 + static void perf_copy_callback(void *data) 175 + { 176 + struct pthr_ctx *pctx = data; 177 + 178 + atomic_dec(&pctx->dma_sync); 179 + } 180 + 181 + static ssize_t perf_copy(struct pthr_ctx *pctx, char *dst, 182 + char *src, size_t size) 183 + { 184 + struct perf_ctx *perf = pctx->perf; 185 + struct dma_async_tx_descriptor *txd; 186 + struct dma_chan *chan = pctx->dma_chan; 187 + struct dma_device *device; 188 + struct dmaengine_unmap_data *unmap; 189 + dma_cookie_t cookie; 190 + size_t src_off, dst_off; 191 + struct perf_mw *mw = &perf->mw; 192 + u64 vbase, dst_vaddr; 193 + dma_addr_t dst_phys; 194 + int retries = 0; 195 + 196 + if (!use_dma) { 197 + memcpy_toio(dst, src, size); 198 + return size; 199 + } 200 + 201 + if (!chan) { 202 + dev_err(&perf->ntb->dev, "DMA engine does not exist\n"); 203 + return -EINVAL; 204 + } 205 + 206 + device = chan->device; 207 + src_off = (size_t)src & ~PAGE_MASK; 208 + dst_off = (size_t)dst & ~PAGE_MASK; 209 + 210 + if (!is_dma_copy_aligned(device, src_off, dst_off, size)) 211 + return -ENODEV; 212 + 213 + vbase = (u64)(u64 *)mw->vbase; 214 + dst_vaddr = (u64)(u64 *)dst; 215 + dst_phys = mw->phys_addr + (dst_vaddr - vbase); 216 + 217 + unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT); 218 + if (!unmap) 219 + return -ENOMEM; 220 + 221 + unmap->len = size; 222 + unmap->addr[0] = dma_map_page(device->dev, virt_to_page(src), 223 + src_off, size, DMA_TO_DEVICE); 224 + if (dma_mapping_error(device->dev, unmap->addr[0])) 225 + goto err_get_unmap; 226 + 227 + unmap->to_cnt = 1; 228 + 229 + do { 230 + txd = device->device_prep_dma_memcpy(chan, dst_phys, 231 + unmap->addr[0], 232 + size, DMA_PREP_INTERRUPT); 233 + if (!txd) { 234 + set_current_state(TASK_INTERRUPTIBLE); 235 + schedule_timeout(DMA_OUT_RESOURCE_TO); 236 + } 237 + } while (!txd && (++retries < DMA_RETRIES)); 238 + 239 + if (!txd) { 240 + pctx->dma_prep_err++; 241 + goto err_get_unmap; 242 + } 243 + 244 + txd->callback = perf_copy_callback; 245 + txd->callback_param = pctx; 246 + dma_set_unmap(txd, unmap); 247 + 248 + cookie = dmaengine_submit(txd); 249 + if (dma_submit_error(cookie)) 250 + goto err_set_unmap; 251 + 252 + atomic_inc(&pctx->dma_sync); 253 + dma_async_issue_pending(chan); 254 + 255 + return size; 256 + 257 + err_set_unmap: 258 + dmaengine_unmap_put(unmap); 259 + err_get_unmap: 260 + dmaengine_unmap_put(unmap); 261 + return 0; 262 + } 263 + 264 + static int perf_move_data(struct pthr_ctx *pctx, char *dst, char *src, 265 + u64 buf_size, u64 win_size, u64 total) 266 + { 267 + int chunks, total_chunks, i; 268 + int copied_chunks = 0; 269 + u64 copied = 0, result; 270 + char *tmp = dst; 271 + u64 perf, diff_us; 272 + ktime_t kstart, kstop, kdiff; 273 + 274 + chunks = div64_u64(win_size, buf_size); 275 + total_chunks = div64_u64(total, buf_size); 276 + kstart = ktime_get(); 277 + 278 + for (i = 0; i < total_chunks; i++) { 279 + result = perf_copy(pctx, tmp, src, buf_size); 280 + copied += result; 281 + copied_chunks++; 282 + if (copied_chunks == chunks) { 283 + tmp = dst; 284 + copied_chunks = 0; 285 + } else 286 + tmp += buf_size; 287 + 288 + /* Probably should schedule every 4GB to prevent soft hang. */ 289 + if (((copied % SZ_4G) == 0) && !use_dma) { 290 + set_current_state(TASK_INTERRUPTIBLE); 291 + schedule_timeout(1); 292 + } 293 + } 294 + 295 + if (use_dma) { 296 + pr_info("%s: All DMA descriptors submitted\n", current->comm); 297 + while (atomic_read(&pctx->dma_sync) != 0) 298 + msleep(20); 299 + } 300 + 301 + kstop = ktime_get(); 302 + kdiff = ktime_sub(kstop, kstart); 303 + diff_us = ktime_to_us(kdiff); 304 + 305 + pr_info("%s: copied %llu bytes\n", current->comm, copied); 306 + 307 + pr_info("%s: lasted %llu usecs\n", current->comm, diff_us); 308 + 309 + perf = div64_u64(copied, diff_us); 310 + 311 + pr_info("%s: MBytes/s: %llu\n", current->comm, perf); 312 + 313 + return 0; 314 + } 315 + 316 + static bool perf_dma_filter_fn(struct dma_chan *chan, void *node) 317 + { 318 + return dev_to_node(&chan->dev->device) == (int)(unsigned long)node; 319 + } 320 + 321 + static int ntb_perf_thread(void *data) 322 + { 323 + struct pthr_ctx *pctx = data; 324 + struct perf_ctx *perf = pctx->perf; 325 + struct pci_dev *pdev = perf->ntb->pdev; 326 + struct perf_mw *mw = &perf->mw; 327 + char *dst; 328 + u64 win_size, buf_size, total; 329 + void *src; 330 + int rc, node, i; 331 + struct dma_chan *dma_chan = NULL; 332 + 333 + pr_info("kthread %s starting...\n", current->comm); 334 + 335 + node = dev_to_node(&pdev->dev); 336 + 337 + if (use_dma && !pctx->dma_chan) { 338 + dma_cap_mask_t dma_mask; 339 + 340 + dma_cap_zero(dma_mask); 341 + dma_cap_set(DMA_MEMCPY, dma_mask); 342 + dma_chan = dma_request_channel(dma_mask, perf_dma_filter_fn, 343 + (void *)(unsigned long)node); 344 + if (!dma_chan) { 345 + pr_warn("%s: cannot acquire DMA channel, quitting\n", 346 + current->comm); 347 + return -ENODEV; 348 + } 349 + pctx->dma_chan = dma_chan; 350 + } 351 + 352 + for (i = 0; i < MAX_SRCS; i++) { 353 + pctx->srcs[i] = kmalloc_node(MAX_TEST_SIZE, GFP_KERNEL, node); 354 + if (!pctx->srcs[i]) { 355 + rc = -ENOMEM; 356 + goto err; 357 + } 358 + } 359 + 360 + win_size = mw->phys_size; 361 + buf_size = 1ULL << seg_order; 362 + total = 1ULL << run_order; 363 + 364 + if (buf_size > MAX_TEST_SIZE) 365 + buf_size = MAX_TEST_SIZE; 366 + 367 + dst = (char *)mw->vbase; 368 + 369 + atomic_inc(&perf->tsync); 370 + while (atomic_read(&perf->tsync) != perf->perf_threads) 371 + schedule(); 372 + 373 + src = pctx->srcs[pctx->src_idx]; 374 + pctx->src_idx = (pctx->src_idx + 1) & (MAX_SRCS - 1); 375 + 376 + rc = perf_move_data(pctx, dst, src, buf_size, win_size, total); 377 + 378 + atomic_dec(&perf->tsync); 379 + 380 + if (rc < 0) { 381 + pr_err("%s: failed\n", current->comm); 382 + rc = -ENXIO; 383 + goto err; 384 + } 385 + 386 + for (i = 0; i < MAX_SRCS; i++) { 387 + kfree(pctx->srcs[i]); 388 + pctx->srcs[i] = NULL; 389 + } 390 + 391 + return 0; 392 + 393 + err: 394 + for (i = 0; i < MAX_SRCS; i++) { 395 + kfree(pctx->srcs[i]); 396 + pctx->srcs[i] = NULL; 397 + } 398 + 399 + if (dma_chan) { 400 + dma_release_channel(dma_chan); 401 + pctx->dma_chan = NULL; 402 + } 403 + 404 + return rc; 405 + } 406 + 407 + static void perf_free_mw(struct perf_ctx *perf) 408 + { 409 + struct perf_mw *mw = &perf->mw; 410 + struct pci_dev *pdev = perf->ntb->pdev; 411 + 412 + if (!mw->virt_addr) 413 + return; 414 + 415 + ntb_mw_clear_trans(perf->ntb, 0); 416 + dma_free_coherent(&pdev->dev, mw->buf_size, 417 + mw->virt_addr, mw->dma_addr); 418 + mw->xlat_size = 0; 419 + mw->buf_size = 0; 420 + mw->virt_addr = NULL; 421 + } 422 + 423 + static int perf_set_mw(struct perf_ctx *perf, resource_size_t size) 424 + { 425 + struct perf_mw *mw = &perf->mw; 426 + size_t xlat_size, buf_size; 427 + 428 + if (!size) 429 + return -EINVAL; 430 + 431 + xlat_size = round_up(size, mw->xlat_align_size); 432 + buf_size = round_up(size, mw->xlat_align); 433 + 434 + if (mw->xlat_size == xlat_size) 435 + return 0; 436 + 437 + if (mw->buf_size) 438 + perf_free_mw(perf); 439 + 440 + mw->xlat_size = xlat_size; 441 + mw->buf_size = buf_size; 442 + 443 + mw->virt_addr = dma_alloc_coherent(&perf->ntb->pdev->dev, buf_size, 444 + &mw->dma_addr, GFP_KERNEL); 445 + if (!mw->virt_addr) { 446 + mw->xlat_size = 0; 447 + mw->buf_size = 0; 448 + } 449 + 450 + return 0; 451 + } 452 + 453 + static void perf_link_work(struct work_struct *work) 454 + { 455 + struct perf_ctx *perf = 456 + container_of(work, struct perf_ctx, link_work.work); 457 + struct ntb_dev *ndev = perf->ntb; 458 + struct pci_dev *pdev = ndev->pdev; 459 + u32 val; 460 + u64 size; 461 + int rc; 462 + 463 + dev_dbg(&perf->ntb->pdev->dev, "%s called\n", __func__); 464 + 465 + size = perf->mw.phys_size; 466 + ntb_peer_spad_write(ndev, MW_SZ_HIGH, upper_32_bits(size)); 467 + ntb_peer_spad_write(ndev, MW_SZ_LOW, lower_32_bits(size)); 468 + ntb_peer_spad_write(ndev, VERSION, PERF_VERSION); 469 + 470 + /* now read what peer wrote */ 471 + val = ntb_spad_read(ndev, VERSION); 472 + if (val != PERF_VERSION) { 473 + dev_dbg(&pdev->dev, "Remote version = %#x\n", val); 474 + goto out; 475 + } 476 + 477 + val = ntb_spad_read(ndev, MW_SZ_HIGH); 478 + size = (u64)val << 32; 479 + 480 + val = ntb_spad_read(ndev, MW_SZ_LOW); 481 + size |= val; 482 + 483 + dev_dbg(&pdev->dev, "Remote MW size = %#llx\n", size); 484 + 485 + rc = perf_set_mw(perf, size); 486 + if (rc) 487 + goto out1; 488 + 489 + perf->link_is_up = true; 490 + 491 + return; 492 + 493 + out1: 494 + perf_free_mw(perf); 495 + 496 + out: 497 + if (ntb_link_is_up(ndev, NULL, NULL) == 1) 498 + schedule_delayed_work(&perf->link_work, 499 + msecs_to_jiffies(PERF_LINK_DOWN_TIMEOUT)); 500 + } 501 + 502 + static void perf_link_cleanup(struct work_struct *work) 503 + { 504 + struct perf_ctx *perf = container_of(work, 505 + struct perf_ctx, 506 + link_cleanup); 507 + 508 + dev_dbg(&perf->ntb->pdev->dev, "%s called\n", __func__); 509 + 510 + if (!perf->link_is_up) 511 + cancel_delayed_work_sync(&perf->link_work); 512 + } 513 + 514 + static int perf_setup_mw(struct ntb_dev *ntb, struct perf_ctx *perf) 515 + { 516 + struct perf_mw *mw; 517 + int rc; 518 + 519 + mw = &perf->mw; 520 + 521 + rc = ntb_mw_get_range(ntb, 0, &mw->phys_addr, &mw->phys_size, 522 + &mw->xlat_align, &mw->xlat_align_size); 523 + if (rc) 524 + return rc; 525 + 526 + perf->mw.vbase = ioremap_wc(mw->phys_addr, mw->phys_size); 527 + if (!mw->vbase) 528 + return -ENOMEM; 529 + 530 + return 0; 531 + } 532 + 533 + static ssize_t debugfs_run_read(struct file *filp, char __user *ubuf, 534 + size_t count, loff_t *offp) 535 + { 536 + struct perf_ctx *perf = filp->private_data; 537 + char *buf; 538 + ssize_t ret, out_offset; 539 + 540 + if (!perf) 541 + return 0; 542 + 543 + buf = kmalloc(64, GFP_KERNEL); 544 + out_offset = snprintf(buf, 64, "%d\n", perf->run); 545 + ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset); 546 + kfree(buf); 547 + 548 + return ret; 549 + } 550 + 551 + static ssize_t debugfs_run_write(struct file *filp, const char __user *ubuf, 552 + size_t count, loff_t *offp) 553 + { 554 + struct perf_ctx *perf = filp->private_data; 555 + int node, i; 556 + 557 + if (!perf->link_is_up) 558 + return 0; 559 + 560 + if (perf->perf_threads == 0) 561 + return 0; 562 + 563 + if (atomic_read(&perf->tsync) == 0) 564 + perf->run = false; 565 + 566 + if (perf->run) { 567 + /* lets stop the threads */ 568 + perf->run = false; 569 + for (i = 0; i < MAX_THREADS; i++) { 570 + if (perf->pthr_ctx[i].thread) { 571 + kthread_stop(perf->pthr_ctx[i].thread); 572 + perf->pthr_ctx[i].thread = NULL; 573 + } else 574 + break; 575 + } 576 + } else { 577 + perf->run = true; 578 + 579 + if (perf->perf_threads > MAX_THREADS) { 580 + perf->perf_threads = MAX_THREADS; 581 + pr_info("Reset total threads to: %u\n", MAX_THREADS); 582 + } 583 + 584 + /* no greater than 1M */ 585 + if (seg_order > MAX_SEG_ORDER) { 586 + seg_order = MAX_SEG_ORDER; 587 + pr_info("Fix seg_order to %u\n", seg_order); 588 + } 589 + 590 + if (run_order < seg_order) { 591 + run_order = seg_order; 592 + pr_info("Fix run_order to %u\n", run_order); 593 + } 594 + 595 + node = dev_to_node(&perf->ntb->pdev->dev); 596 + /* launch kernel thread */ 597 + for (i = 0; i < perf->perf_threads; i++) { 598 + struct pthr_ctx *pctx; 599 + 600 + pctx = &perf->pthr_ctx[i]; 601 + atomic_set(&pctx->dma_sync, 0); 602 + pctx->perf = perf; 603 + pctx->thread = 604 + kthread_create_on_node(ntb_perf_thread, 605 + (void *)pctx, 606 + node, "ntb_perf %d", i); 607 + if (pctx->thread) 608 + wake_up_process(pctx->thread); 609 + else { 610 + perf->run = false; 611 + for (i = 0; i < MAX_THREADS; i++) { 612 + if (pctx->thread) { 613 + kthread_stop(pctx->thread); 614 + pctx->thread = NULL; 615 + } 616 + } 617 + } 618 + 619 + if (perf->run == false) 620 + return -ENXIO; 621 + } 622 + 623 + } 624 + 625 + return count; 626 + } 627 + 628 + static const struct file_operations ntb_perf_debugfs_run = { 629 + .owner = THIS_MODULE, 630 + .open = simple_open, 631 + .read = debugfs_run_read, 632 + .write = debugfs_run_write, 633 + }; 634 + 635 + static int perf_debugfs_setup(struct perf_ctx *perf) 636 + { 637 + struct pci_dev *pdev = perf->ntb->pdev; 638 + 639 + if (!debugfs_initialized()) 640 + return -ENODEV; 641 + 642 + if (!perf_debugfs_dir) { 643 + perf_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 644 + if (!perf_debugfs_dir) 645 + return -ENODEV; 646 + } 647 + 648 + perf->debugfs_node_dir = debugfs_create_dir(pci_name(pdev), 649 + perf_debugfs_dir); 650 + if (!perf->debugfs_node_dir) 651 + return -ENODEV; 652 + 653 + perf->debugfs_run = debugfs_create_file("run", S_IRUSR | S_IWUSR, 654 + perf->debugfs_node_dir, perf, 655 + &ntb_perf_debugfs_run); 656 + if (!perf->debugfs_run) 657 + return -ENODEV; 658 + 659 + perf->debugfs_threads = debugfs_create_u8("threads", S_IRUSR | S_IWUSR, 660 + perf->debugfs_node_dir, 661 + &perf->perf_threads); 662 + if (!perf->debugfs_threads) 663 + return -ENODEV; 664 + 665 + return 0; 666 + } 667 + 668 + static int perf_probe(struct ntb_client *client, struct ntb_dev *ntb) 669 + { 670 + struct pci_dev *pdev = ntb->pdev; 671 + struct perf_ctx *perf; 672 + int node; 673 + int rc = 0; 674 + 675 + node = dev_to_node(&pdev->dev); 676 + 677 + perf = kzalloc_node(sizeof(*perf), GFP_KERNEL, node); 678 + if (!perf) { 679 + rc = -ENOMEM; 680 + goto err_perf; 681 + } 682 + 683 + perf->ntb = ntb; 684 + perf->perf_threads = 1; 685 + atomic_set(&perf->tsync, 0); 686 + perf->run = false; 687 + spin_lock_init(&perf->db_lock); 688 + perf_setup_mw(ntb, perf); 689 + INIT_DELAYED_WORK(&perf->link_work, perf_link_work); 690 + INIT_WORK(&perf->link_cleanup, perf_link_cleanup); 691 + 692 + rc = ntb_set_ctx(ntb, perf, &perf_ops); 693 + if (rc) 694 + goto err_ctx; 695 + 696 + perf->link_is_up = false; 697 + ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 698 + ntb_link_event(ntb); 699 + 700 + rc = perf_debugfs_setup(perf); 701 + if (rc) 702 + goto err_ctx; 703 + 704 + return 0; 705 + 706 + err_ctx: 707 + cancel_delayed_work_sync(&perf->link_work); 708 + cancel_work_sync(&perf->link_cleanup); 709 + kfree(perf); 710 + err_perf: 711 + return rc; 712 + } 713 + 714 + static void perf_remove(struct ntb_client *client, struct ntb_dev *ntb) 715 + { 716 + struct perf_ctx *perf = ntb->ctx; 717 + int i; 718 + 719 + dev_dbg(&perf->ntb->dev, "%s called\n", __func__); 720 + 721 + cancel_delayed_work_sync(&perf->link_work); 722 + cancel_work_sync(&perf->link_cleanup); 723 + 724 + ntb_clear_ctx(ntb); 725 + ntb_link_disable(ntb); 726 + 727 + debugfs_remove_recursive(perf_debugfs_dir); 728 + perf_debugfs_dir = NULL; 729 + 730 + if (use_dma) { 731 + for (i = 0; i < MAX_THREADS; i++) { 732 + struct pthr_ctx *pctx = &perf->pthr_ctx[i]; 733 + 734 + if (pctx->dma_chan) 735 + dma_release_channel(pctx->dma_chan); 736 + } 737 + } 738 + 739 + kfree(perf); 740 + } 741 + 742 + static struct ntb_client perf_client = { 743 + .ops = { 744 + .probe = perf_probe, 745 + .remove = perf_remove, 746 + }, 747 + }; 748 + module_ntb_client(perf_client);

Configure Feed

Configure Feed