Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'for-linus' of git://git.kernel.dk/linux-block
Pull NVMe fix from Jens Axboe:
"Late addition here, it's basically a revert of a patch that was added
in this merge window, but has proven to cause problems.
This is swapping out the RCU based namespace protection with a good
old mutex instead"
* 'for-linus' of git://git.kernel.dk/linux-block:
nvme: Remove RCU namespace protection
+39
-35
+39
-35
drivers/nvme/host/core.c
+39
-35
drivers/nvme/host/core.c
···
1394
1394
return nsa->ns_id - nsb->ns_id;
1395
1395
}
1396
1396
1397
-
static struct nvme_ns *nvme_find_ns(struct nvme_ctrl *ctrl, unsigned nsid)
1397
+
static struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
1398
1398
{
1399
-
struct nvme_ns *ns;
1399
+
struct nvme_ns *ns, *ret = NULL;
1400
1400
1401
-
lockdep_assert_held(&ctrl->namespaces_mutex);
1402
-
1401
+
mutex_lock(&ctrl->namespaces_mutex);
1403
1402
list_for_each_entry(ns, &ctrl->namespaces, list) {
1404
-
if (ns->ns_id == nsid)
1405
-
return ns;
1403
+
if (ns->ns_id == nsid) {
1404
+
kref_get(&ns->kref);
1405
+
ret = ns;
1406
+
break;
1407
+
}
1406
1408
if (ns->ns_id > nsid)
1407
1409
break;
1408
1410
}
1409
-
return NULL;
1411
+
mutex_unlock(&ctrl->namespaces_mutex);
1412
+
return ret;
1410
1413
}
1411
1414
1412
1415
static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
···
1417
1414
struct nvme_ns *ns;
1418
1415
struct gendisk *disk;
1419
1416
int node = dev_to_node(ctrl->dev);
1420
-
1421
-
lockdep_assert_held(&ctrl->namespaces_mutex);
1422
1417
1423
1418
ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
1424
1419
if (!ns)
···
1458
1457
if (nvme_revalidate_disk(ns->disk))
1459
1458
goto out_free_disk;
1460
1459
1461
-
list_add_tail_rcu(&ns->list, &ctrl->namespaces);
1460
+
mutex_lock(&ctrl->namespaces_mutex);
1461
+
list_add_tail(&ns->list, &ctrl->namespaces);
1462
+
mutex_unlock(&ctrl->namespaces_mutex);
1463
+
1462
1464
kref_get(&ctrl->kref);
1463
1465
if (ns->type == NVME_NS_LIGHTNVM)
1464
1466
return;
···
1484
1480
1485
1481
static void nvme_ns_remove(struct nvme_ns *ns)
1486
1482
{
1487
-
lockdep_assert_held(&ns->ctrl->namespaces_mutex);
1488
-
1489
1483
if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
1490
1484
return;
1491
1485
···
1496
1494
blk_mq_abort_requeue_list(ns->queue);
1497
1495
blk_cleanup_queue(ns->queue);
1498
1496
}
1497
+
1498
+
mutex_lock(&ns->ctrl->namespaces_mutex);
1499
1499
list_del_init(&ns->list);
1500
-
synchronize_rcu();
1500
+
mutex_unlock(&ns->ctrl->namespaces_mutex);
1501
+
1501
1502
nvme_put_ns(ns);
1502
1503
}
1503
1504
···
1508
1503
{
1509
1504
struct nvme_ns *ns;
1510
1505
1511
-
ns = nvme_find_ns(ctrl, nsid);
1506
+
ns = nvme_find_get_ns(ctrl, nsid);
1512
1507
if (ns) {
1513
1508
if (revalidate_disk(ns->disk))
1514
1509
nvme_ns_remove(ns);
1510
+
nvme_put_ns(ns);
1515
1511
} else
1516
1512
nvme_alloc_ns(ctrl, nsid);
1517
1513
}
···
1541
1535
nvme_validate_ns(ctrl, nsid);
1542
1536
1543
1537
while (++prev < nsid) {
1544
-
ns = nvme_find_ns(ctrl, prev);
1545
-
if (ns)
1538
+
ns = nvme_find_get_ns(ctrl, prev);
1539
+
if (ns) {
1546
1540
nvme_ns_remove(ns);
1541
+
nvme_put_ns(ns);
1542
+
}
1547
1543
}
1548
1544
}
1549
1545
nn -= j;
···
1559
1551
{
1560
1552
struct nvme_ns *ns, *next;
1561
1553
unsigned i;
1562
-
1563
-
lockdep_assert_held(&ctrl->namespaces_mutex);
1564
1554
1565
1555
for (i = 1; i <= nn; i++)
1566
1556
nvme_validate_ns(ctrl, i);
···
1582
1576
if (nvme_identify_ctrl(ctrl, &id))
1583
1577
return;
1584
1578
1585
-
mutex_lock(&ctrl->namespaces_mutex);
1586
1579
nn = le32_to_cpu(id->nn);
1587
1580
if (ctrl->vs >= NVME_VS(1, 1) &&
1588
1581
!(ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)) {
···
1590
1585
}
1591
1586
nvme_scan_ns_sequential(ctrl, nn);
1592
1587
done:
1588
+
mutex_lock(&ctrl->namespaces_mutex);
1593
1589
list_sort(NULL, &ctrl->namespaces, ns_cmp);
1594
1590
mutex_unlock(&ctrl->namespaces_mutex);
1595
1591
kfree(id);
···
1610
1604
}
1611
1605
EXPORT_SYMBOL_GPL(nvme_queue_scan);
1612
1606
1607
+
/*
1608
+
* This function iterates the namespace list unlocked to allow recovery from
1609
+
* controller failure. It is up to the caller to ensure the namespace list is
1610
+
* not modified by scan work while this function is executing.
1611
+
*/
1613
1612
void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
1614
1613
{
1615
1614
struct nvme_ns *ns, *next;
···
1628
1617
if (ctrl->state == NVME_CTRL_DEAD)
1629
1618
nvme_kill_queues(ctrl);
1630
1619
1631
-
mutex_lock(&ctrl->namespaces_mutex);
1632
1620
list_for_each_entry_safe(ns, next, &ctrl->namespaces, list)
1633
1621
nvme_ns_remove(ns);
1634
-
mutex_unlock(&ctrl->namespaces_mutex);
1635
1622
}
1636
1623
EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
1637
1624
···
1800
1791
{
1801
1792
struct nvme_ns *ns;
1802
1793
1803
-
rcu_read_lock();
1804
-
list_for_each_entry_rcu(ns, &ctrl->namespaces, list) {
1805
-
if (!kref_get_unless_zero(&ns->kref))
1806
-
continue;
1807
-
1794
+
mutex_lock(&ctrl->namespaces_mutex);
1795
+
list_for_each_entry(ns, &ctrl->namespaces, list) {
1808
1796
/*
1809
1797
* Revalidating a dead namespace sets capacity to 0. This will
1810
1798
* end buffered writers dirtying pages that can't be synced.
···
1812
1806
blk_set_queue_dying(ns->queue);
1813
1807
blk_mq_abort_requeue_list(ns->queue);
1814
1808
blk_mq_start_stopped_hw_queues(ns->queue, true);
1815
-
1816
-
nvme_put_ns(ns);
1817
1809
}
1818
-
rcu_read_unlock();
1810
+
mutex_unlock(&ctrl->namespaces_mutex);
1819
1811
}
1820
1812
EXPORT_SYMBOL_GPL(nvme_kill_queues);
1821
1813
···
1821
1817
{
1822
1818
struct nvme_ns *ns;
1823
1819
1824
-
rcu_read_lock();
1825
-
list_for_each_entry_rcu(ns, &ctrl->namespaces, list) {
1820
+
mutex_lock(&ctrl->namespaces_mutex);
1821
+
list_for_each_entry(ns, &ctrl->namespaces, list) {
1826
1822
spin_lock_irq(ns->queue->queue_lock);
1827
1823
queue_flag_set(QUEUE_FLAG_STOPPED, ns->queue);
1828
1824
spin_unlock_irq(ns->queue->queue_lock);
···
1830
1826
blk_mq_cancel_requeue_work(ns->queue);
1831
1827
blk_mq_stop_hw_queues(ns->queue);
1832
1828
}
1833
-
rcu_read_unlock();
1829
+
mutex_unlock(&ctrl->namespaces_mutex);
1834
1830
}
1835
1831
EXPORT_SYMBOL_GPL(nvme_stop_queues);
1836
1832
···
1838
1834
{
1839
1835
struct nvme_ns *ns;
1840
1836
1841
-
rcu_read_lock();
1842
-
list_for_each_entry_rcu(ns, &ctrl->namespaces, list) {
1837
+
mutex_lock(&ctrl->namespaces_mutex);
1838
+
list_for_each_entry(ns, &ctrl->namespaces, list) {
1843
1839
queue_flag_clear_unlocked(QUEUE_FLAG_STOPPED, ns->queue);
1844
1840
blk_mq_start_stopped_hw_queues(ns->queue, true);
1845
1841
blk_mq_kick_requeue_list(ns->queue);
1846
1842
}
1847
-
rcu_read_unlock();
1843
+
mutex_unlock(&ctrl->namespaces_mutex);
1848
1844
}
1849
1845
EXPORT_SYMBOL_GPL(nvme_start_queues);
1850
1846