net: mana: Trigger VF reset/recovery on health check failure due to HWC timeout

The GF stats periodic query is used as mechanism to monitor HWC health
check. If this HWC command times out, it is a strong indication that
the device/SoC is in a faulty state and requires recovery.

Today, when a timeout is detected, the driver marks
hwc_timeout_occurred, clears cached stats, and stops rescheduling the
periodic work. However, the device itself is left in the same failing
state.

Extend the timeout handling path to trigger the existing MANA VF
recovery service by queueing a GDMA_EQE_HWC_RESET_REQUEST work item.
This is expected to initiate the appropriate recovery flow by suspende
resume first and if it fails then trigger a bus rescan.

This change is intentionally limited to HWC command timeouts and does
not trigger recovery for errors reported by the SoC as a normal command
response.

Signed-off-by: Dipayaan Roy <dipayanroy@linux.microsoft.com>
Reviewed-by: Haiyang Zhang <haiyangz@microsoft.com>
Reviewed-by: Simon Horman <horms@kernel.org>
Link: https://patch.msgid.link/aaFShvKnwR5FY8dH@linuxonhyperv3.guj3yctzbm1etfxqx2vob5hsef.xx.internal.cloudapp.net
Signed-off-by: Paolo Abeni <pabeni@redhat.com>

authored by

Dipayaan Roy and committed by

Paolo Abeni 4 months ago 2b12ffb6 c69855ad

+55 -35

3 changed files

expand all

drivers

net

ethernet

microsoft

mana

gdma_main.c

mana_en.c

include

net

mana

gdma.h

+33 -32

drivers/net/ethernet/microsoft/mana/gdma_main.c

··· 490 490 dev_info(&pdev->dev, "MANA reset cycle completed\n"); 491 491 492 492 out: 493 - gc->in_service = false; 493 + clear_bit(GC_IN_SERVICE, &gc->flags); 494 494 } 495 - 496 - struct mana_serv_work { 497 - struct work_struct serv_work; 498 - struct pci_dev *pdev; 499 - enum gdma_eqe_type type; 500 - }; 501 495 502 496 static void mana_do_service(enum gdma_eqe_type type, struct pci_dev *pdev) 503 497 { ··· 552 558 module_put(THIS_MODULE); 553 559 } 554 560 561 + int mana_schedule_serv_work(struct gdma_context *gc, enum gdma_eqe_type type) 562 + { 563 + struct mana_serv_work *mns_wk; 564 + 565 + if (test_and_set_bit(GC_IN_SERVICE, &gc->flags)) { 566 + dev_info(gc->dev, "Already in service\n"); 567 + return -EBUSY; 568 + } 569 + 570 + if (!try_module_get(THIS_MODULE)) { 571 + dev_info(gc->dev, "Module is unloading\n"); 572 + clear_bit(GC_IN_SERVICE, &gc->flags); 573 + return -ENODEV; 574 + } 575 + 576 + mns_wk = kzalloc(sizeof(*mns_wk), GFP_ATOMIC); 577 + if (!mns_wk) { 578 + module_put(THIS_MODULE); 579 + clear_bit(GC_IN_SERVICE, &gc->flags); 580 + return -ENOMEM; 581 + } 582 + 583 + dev_info(gc->dev, "Start MANA service type:%d\n", type); 584 + mns_wk->pdev = to_pci_dev(gc->dev); 585 + mns_wk->type = type; 586 + pci_dev_get(mns_wk->pdev); 587 + INIT_WORK(&mns_wk->serv_work, mana_serv_func); 588 + schedule_work(&mns_wk->serv_work); 589 + return 0; 590 + } 591 + 555 592 static void mana_gd_process_eqe(struct gdma_queue *eq) 556 593 { 557 594 u32 head = eq->head % (eq->queue_size / GDMA_EQE_SIZE); 558 595 struct gdma_context *gc = eq->gdma_dev->gdma_context; 559 596 struct gdma_eqe *eq_eqe_ptr = eq->queue_mem_ptr; 560 - struct mana_serv_work *mns_wk; 561 597 union gdma_eqe_info eqe_info; 562 598 enum gdma_eqe_type type; 563 599 struct gdma_event event; ··· 647 623 "Service is to be processed in probe\n"); 648 624 break; 649 625 } 650 - 651 - if (gc->in_service) { 652 - dev_info(gc->dev, "Already in service\n"); 653 - break; 654 - } 655 - 656 - if (!try_module_get(THIS_MODULE)) { 657 - dev_info(gc->dev, "Module is unloading\n"); 658 - break; 659 - } 660 - 661 - mns_wk = kzalloc_obj(*mns_wk, GFP_ATOMIC); 662 - if (!mns_wk) { 663 - module_put(THIS_MODULE); 664 - break; 665 - } 666 - 667 - dev_info(gc->dev, "Start MANA service type:%d\n", type); 668 - gc->in_service = true; 669 - mns_wk->pdev = to_pci_dev(gc->dev); 670 - mns_wk->type = type; 671 - pci_dev_get(mns_wk->pdev); 672 - INIT_WORK(&mns_wk->serv_work, mana_serv_func); 673 - schedule_work(&mns_wk->serv_work); 626 + mana_schedule_serv_work(gc, type); 674 627 break; 675 628 676 629 default:

+8 -1

drivers/net/ethernet/microsoft/mana/mana_en.c

··· 875 875 struct gdma_context *gc = ac->gdma_dev->gdma_context; 876 876 877 877 /* Already in service, hence tx queue reset is not required.*/ 878 - if (gc->in_service) 878 + if (test_bit(GC_IN_SERVICE, &gc->flags)) 879 879 return; 880 880 881 881 /* Note: If there are pending queue reset work for this port(apc), ··· 3525 3525 { 3526 3526 struct mana_context *ac = 3527 3527 container_of(to_delayed_work(work), struct mana_context, gf_stats_work); 3528 + struct gdma_context *gc = ac->gdma_dev->gdma_context; 3528 3529 int err; 3529 3530 3530 3531 err = mana_query_gf_stats(ac); ··· 3533 3532 /* HWC timeout detected - reset stats and stop rescheduling */ 3534 3533 ac->hwc_timeout_occurred = true; 3535 3534 memset(&ac->hc_stats, 0, sizeof(ac->hc_stats)); 3535 + dev_warn(gc->dev, 3536 + "Gf stats wk handler: gf stats query timed out.\n"); 3537 + /* As HWC timed out, indicating a faulty HW state and needs a 3538 + * reset. 3539 + */ 3540 + mana_schedule_serv_work(gc, GDMA_EQE_HWC_RESET_REQUEST); 3536 3541 return; 3537 3542 } 3538 3543 schedule_delayed_work(&ac->gf_stats_work, MANA_GF_STATS_PERIOD);

+14 -2

include/net/mana/gdma.h

··· 215 215 216 216 #define GDMA_INVALID_DMA_REGION 0 217 217 218 + struct mana_serv_work { 219 + struct work_struct serv_work; 220 + struct pci_dev *pdev; 221 + enum gdma_eqe_type type; 222 + }; 223 + 218 224 struct gdma_mem_info { 219 225 struct device *dev; 220 226 ··· 392 386 393 387 enum gdma_context_flags { 394 388 GC_PROBE_SUCCEEDED = 0, 389 + GC_IN_SERVICE = 1, 395 390 }; 396 391 397 392 struct gdma_context { ··· 418 411 u32 test_event_eq_id; 419 412 420 413 bool is_pf; 421 - bool in_service; 422 414 423 415 phys_addr_t bar0_pa; 424 416 void __iomem *bar0_va; ··· 478 472 int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe); 479 473 480 474 void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit); 475 + 476 + int mana_schedule_serv_work(struct gdma_context *gc, enum gdma_eqe_type type); 481 477 482 478 struct gdma_wqe { 483 479 u32 reserved :24; ··· 623 615 /* Driver can handle hardware recovery events during probe */ 624 616 #define GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY BIT(22) 625 617 618 + /* Driver supports self recovery on Hardware Channel timeouts */ 619 + #define GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY BIT(25) 620 + 626 621 #define GDMA_DRV_CAP_FLAGS1 \ 627 622 (GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT | \ 628 623 GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX | \ ··· 639 628 GDMA_DRV_CAP_FLAG_1_PERIODIC_STATS_QUERY | \ 640 629 GDMA_DRV_CAP_FLAG_1_SKB_LINEARIZE | \ 641 630 GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY | \ 642 - GDMA_DRV_CAP_FLAG_1_HANDLE_STALL_SQ_RECOVERY) 631 + GDMA_DRV_CAP_FLAG_1_HANDLE_STALL_SQ_RECOVERY | \ 632 + GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY) 643 633 644 634 #define GDMA_DRV_CAP_FLAGS2 0 645 635

Configure Feed

Configure Feed