drm/amdgpu: add an option to allow gpu partition allocate all available memory

Current driver reports and limits memory allocation for each partition equally
among partitions using same memory partition. Application may not be able to
use all available memory when run on a partitioned gpu though system still has
enough free memory.

Add an option that app can use to have gpu partition allocate all available
memory.

Signed-off-by: Xiaogang Chen <xiaogang.chen@amd.com>
Reviewed-by: Philip Yang <philip.yang@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

authored by

Xiaogang Chen and committed by

Alex Deucher 2 months ago e0e9792e f315099f

+63 -1

5 changed files

expand all

drivers

gpu

drm

amd

amdgpu

amdgpu_amdkfd.c

amdgpu_gfx.c

amdgpu_gfx.h

amdgpu_xcp.c

amdgpu_xcp.h

+4 -1

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c

··· 805 805 } else { 806 806 tmp = adev->gmc.mem_partitions[mem_id].size; 807 807 } 808 - do_div(tmp, adev->xcp_mgr->num_xcp_per_mem_partition); 808 + 809 + if (adev->xcp_mgr->mem_alloc_mode == AMDGPU_PARTITION_MEM_CAPPING_EVEN) 810 + do_div(tmp, adev->xcp_mgr->num_xcp_per_mem_partition); 811 + 809 812 return ALIGN_DOWN(tmp, PAGE_SIZE); 810 813 } else if (adev->apu_prefer_gtt) { 811 814 return (ttm_tt_pages_limit() << PAGE_SHIFT);

+39

drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c

··· 1580 1580 return count; 1581 1581 } 1582 1582 1583 + static ssize_t compute_partition_mem_alloc_mode_show(struct device *dev, 1584 + struct device_attribute *addr, 1585 + char *buf) 1586 + { 1587 + struct drm_device *ddev = dev_get_drvdata(dev); 1588 + struct amdgpu_device *adev = drm_to_adev(ddev); 1589 + int mode = adev->xcp_mgr->mem_alloc_mode; 1590 + 1591 + return sysfs_emit(buf, "%s\n", 1592 + amdgpu_gfx_compute_mem_alloc_mode_desc(mode)); 1593 + } 1594 + 1595 + 1596 + static ssize_t compute_partition_mem_alloc_mode_store(struct device *dev, 1597 + struct device_attribute *addr, 1598 + const char *buf, size_t count) 1599 + { 1600 + struct drm_device *ddev = dev_get_drvdata(dev); 1601 + struct amdgpu_device *adev = drm_to_adev(ddev); 1602 + 1603 + if (!strncasecmp("CAPPING", buf, strlen("CAPPING"))) 1604 + adev->xcp_mgr->mem_alloc_mode = AMDGPU_PARTITION_MEM_CAPPING_EVEN; 1605 + else if (!strncasecmp("ALL", buf, strlen("ALL"))) 1606 + adev->xcp_mgr->mem_alloc_mode = AMDGPU_PARTITION_MEM_ALLOC_ALL; 1607 + else 1608 + return -EINVAL; 1609 + 1610 + return count; 1611 + } 1612 + 1583 1613 static const char *xcp_desc[] = { 1584 1614 [AMDGPU_SPX_PARTITION_MODE] = "SPX", 1585 1615 [AMDGPU_DPX_PARTITION_MODE] = "DPX", ··· 1965 1935 static DEVICE_ATTR(compute_reset_mask, 0444, 1966 1936 amdgpu_gfx_get_compute_reset_mask, NULL); 1967 1937 1938 + static DEVICE_ATTR(compute_partition_mem_alloc_mode, 0644, 1939 + compute_partition_mem_alloc_mode_show, 1940 + compute_partition_mem_alloc_mode_store); 1941 + 1968 1942 static int amdgpu_gfx_sysfs_xcp_init(struct amdgpu_device *adev) 1969 1943 { 1970 1944 struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr; ··· 1986 1952 ~(S_IWUSR | S_IWGRP | S_IWOTH); 1987 1953 1988 1954 r = device_create_file(adev->dev, &dev_attr_current_compute_partition); 1955 + if (r) 1956 + return r; 1957 + 1958 + r = device_create_file(adev->dev, 1959 + &dev_attr_compute_partition_mem_alloc_mode); 1989 1960 if (r) 1990 1961 return r; 1991 1962

+17

drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h

··· 71 71 AMDGPU_AUTO_COMPUTE_PARTITION_MODE = -2, 72 72 }; 73 73 74 + enum amdgpu_gfx_partition_mem_alloc_mode { 75 + AMDGPU_PARTITION_MEM_CAPPING_EVEN = 0, 76 + AMDGPU_PARTITION_MEM_ALLOC_ALL = 1, 77 + }; 78 + 74 79 #define NUM_XCC(x) hweight16(x) 75 80 76 81 enum amdgpu_gfx_ras_mem_id_type { ··· 677 672 return "QPX"; 678 673 case AMDGPU_CPX_PARTITION_MODE: 679 674 return "CPX"; 675 + default: 676 + return "UNKNOWN"; 677 + } 678 + } 679 + 680 + static inline const char *amdgpu_gfx_compute_mem_alloc_mode_desc(int mode) 681 + { 682 + switch (mode) { 683 + case AMDGPU_PARTITION_MEM_CAPPING_EVEN: 684 + return "CAPPING"; 685 + case AMDGPU_PARTITION_MEM_ALLOC_ALL: 686 + return "ALL"; 680 687 default: 681 688 return "UNKNOWN"; 682 689 }

drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c

··· 181 181 } 182 182 183 183 xcp_mgr->num_xcps = num_xcps; 184 + xcp_mgr->mem_alloc_mode = AMDGPU_PARTITION_MEM_CAPPING_EVEN; 184 185 amdgpu_xcp_update_partition_sched_list(adev); 185 186 186 187 return 0;

drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.h

··· 132 132 struct amdgpu_xcp_cfg *xcp_cfg; 133 133 uint32_t supp_xcp_modes; 134 134 uint32_t avail_xcp_modes; 135 + /* used to determin KFD memory alloc mode for each partition */ 136 + uint32_t mem_alloc_mode; 135 137 }; 136 138 137 139 struct amdgpu_xcp_mgr_funcs {

Configure Feed

Configure Feed