drm/ttm, drm_xe, Implement ttm_lru_walk_for_evict() using the guarded LRU iteration

+75 -113

drivers/gpu/drm/ttm/ttm_bo_util.c

··· 773 773 return ret; 774 774 } 775 775 776 - static bool ttm_lru_walk_trylock(struct ttm_lru_walk_arg *arg, 777 - struct ttm_buffer_object *bo, 778 - bool *needs_unlock) 776 + static bool ttm_lru_walk_trylock(struct ttm_bo_lru_cursor *curs, 777 + struct ttm_buffer_object *bo) 779 778 { 780 - struct ttm_operation_ctx *ctx = arg->ctx; 779 + struct ttm_operation_ctx *ctx = curs->arg->ctx; 781 780 782 - *needs_unlock = false; 781 + curs->needs_unlock = false; 783 782 784 783 if (dma_resv_trylock(bo->base.resv)) { 785 - *needs_unlock = true; 784 + curs->needs_unlock = true; 786 785 return true; 787 786 } 788 787 ··· 793 794 return false; 794 795 } 795 796 796 - static int ttm_lru_walk_ticketlock(struct ttm_lru_walk_arg *arg, 797 - struct ttm_buffer_object *bo, 798 - bool *needs_unlock) 797 + static int ttm_lru_walk_ticketlock(struct ttm_bo_lru_cursor *curs, 798 + struct ttm_buffer_object *bo) 799 799 { 800 + struct ttm_lru_walk_arg *arg = curs->arg; 800 801 struct dma_resv *resv = bo->base.resv; 801 802 int ret; 802 803 ··· 806 807 ret = dma_resv_lock(resv, arg->ticket); 807 808 808 809 if (!ret) { 809 - *needs_unlock = true; 810 + curs->needs_unlock = true; 810 811 /* 811 812 * Only a single ticketlock per loop. Ticketlocks are prone 812 813 * to return -EDEADLK causing the eviction to fail, so ··· 820 821 } 821 822 822 823 return ret; 823 - } 824 - 825 - static void ttm_lru_walk_unlock(struct ttm_buffer_object *bo, bool locked) 826 - { 827 - if (locked) 828 - dma_resv_unlock(bo->base.resv); 829 824 } 830 825 831 826 /** ··· 856 863 s64 ttm_lru_walk_for_evict(struct ttm_lru_walk *walk, struct ttm_device *bdev, 857 864 struct ttm_resource_manager *man, s64 target) 858 865 { 859 - struct ttm_resource_cursor cursor; 860 - struct ttm_resource *res; 866 + struct ttm_bo_lru_cursor cursor; 867 + struct ttm_buffer_object *bo; 861 868 s64 progress = 0; 862 869 s64 lret; 863 870 864 - spin_lock(&bdev->lru_lock); 865 - ttm_resource_cursor_init(&cursor, man); 866 - ttm_resource_manager_for_each_res(&cursor, res) { 867 - struct ttm_buffer_object *bo = res->bo; 868 - bool bo_needs_unlock = false; 869 - bool bo_locked = false; 870 - int mem_type; 871 - 872 - /* 873 - * Attempt a trylock before taking a reference on the bo, 874 - * since if we do it the other way around, and the trylock fails, 875 - * we need to drop the lru lock to put the bo. 876 - */ 877 - if (ttm_lru_walk_trylock(&walk->arg, bo, &bo_needs_unlock)) 878 - bo_locked = true; 879 - else if (!walk->arg.ticket || walk->arg.ctx->no_wait_gpu || 880 - walk->arg.trylock_only) 881 - continue; 882 - 883 - if (!ttm_bo_get_unless_zero(bo)) { 884 - ttm_lru_walk_unlock(bo, bo_needs_unlock); 885 - continue; 886 - } 887 - 888 - mem_type = res->mem_type; 889 - spin_unlock(&bdev->lru_lock); 890 - 891 - lret = 0; 892 - if (!bo_locked) 893 - lret = ttm_lru_walk_ticketlock(&walk->arg, bo, &bo_needs_unlock); 894 - 895 - /* 896 - * Note that in between the release of the lru lock and the 897 - * ticketlock, the bo may have switched resource, 898 - * and also memory type, since the resource may have been 899 - * freed and allocated again with a different memory type. 900 - * In that case, just skip it. 901 - */ 902 - if (!lret && bo->resource && bo->resource->mem_type == mem_type) 903 - lret = walk->ops->process_bo(walk, bo); 904 - 905 - ttm_lru_walk_unlock(bo, bo_needs_unlock); 906 - ttm_bo_put(bo); 871 + ttm_bo_lru_for_each_reserved_guarded(&cursor, man, &walk->arg, bo) { 872 + lret = walk->ops->process_bo(walk, bo); 907 873 if (lret == -EBUSY || lret == -EALREADY) 908 874 lret = 0; 909 875 progress = (lret < 0) ? lret : progress + lret; 910 - 911 - spin_lock(&bdev->lru_lock); 912 876 if (progress < 0 || progress >= target) 913 877 break; 914 878 } 915 - ttm_resource_cursor_fini(&cursor); 916 - spin_unlock(&bdev->lru_lock); 879 + if (IS_ERR(bo)) 880 + return PTR_ERR(bo); 917 881 918 882 return progress; 919 883 } ··· 910 960 * @man: The ttm resource_manager whose LRU lists to iterate over. 911 961 * @arg: The ttm_lru_walk_arg to govern the walk. 912 962 * 913 - * Initialize a struct ttm_bo_lru_cursor. Currently only trylocking 914 - * or prelocked buffer objects are available as detailed by 915 - * @arg->ctx.resv and @arg->ctx.allow_res_evict. Ticketlocking is not 916 - * supported. 963 + * Initialize a struct ttm_bo_lru_cursor. 917 964 * 918 965 * Return: Pointer to @curs. The function does not fail. 919 966 */ ··· 928 981 EXPORT_SYMBOL(ttm_bo_lru_cursor_init); 929 982 930 983 static struct ttm_buffer_object * 931 - ttm_bo_from_res_reserved(struct ttm_resource *res, struct ttm_bo_lru_cursor *curs) 984 + __ttm_bo_lru_cursor_next(struct ttm_bo_lru_cursor *curs) 932 985 { 933 - struct ttm_buffer_object *bo = res->bo; 986 + spinlock_t *lru_lock = &curs->res_curs.man->bdev->lru_lock; 987 + struct ttm_resource *res = NULL; 988 + struct ttm_buffer_object *bo; 989 + struct ttm_lru_walk_arg *arg = curs->arg; 990 + bool first = !curs->bo; 934 991 935 - if (!ttm_lru_walk_trylock(curs->arg, bo, &curs->needs_unlock)) 936 - return NULL; 992 + ttm_bo_lru_cursor_cleanup_bo(curs); 937 993 938 - if (!ttm_bo_get_unless_zero(bo)) { 939 - if (curs->needs_unlock) 940 - dma_resv_unlock(bo->base.resv); 941 - return NULL; 994 + spin_lock(lru_lock); 995 + for (;;) { 996 + int mem_type, ret = 0; 997 + bool bo_locked = false; 998 + 999 + if (first) { 1000 + res = ttm_resource_manager_first(&curs->res_curs); 1001 + first = false; 1002 + } else { 1003 + res = ttm_resource_manager_next(&curs->res_curs); 1004 + } 1005 + if (!res) 1006 + break; 1007 + 1008 + bo = res->bo; 1009 + if (ttm_lru_walk_trylock(curs, bo)) 1010 + bo_locked = true; 1011 + else if (!arg->ticket || arg->ctx->no_wait_gpu || arg->trylock_only) 1012 + continue; 1013 + 1014 + if (!ttm_bo_get_unless_zero(bo)) { 1015 + if (curs->needs_unlock) 1016 + dma_resv_unlock(bo->base.resv); 1017 + continue; 1018 + } 1019 + 1020 + mem_type = res->mem_type; 1021 + spin_unlock(lru_lock); 1022 + if (!bo_locked) 1023 + ret = ttm_lru_walk_ticketlock(curs, bo); 1024 + 1025 + /* 1026 + * Note that in between the release of the lru lock and the 1027 + * ticketlock, the bo may have switched resource, 1028 + * and also memory type, since the resource may have been 1029 + * freed and allocated again with a different memory type. 1030 + * In that case, just skip it. 1031 + */ 1032 + curs->bo = bo; 1033 + if (!ret && bo->resource && bo->resource->mem_type == mem_type) 1034 + return bo; 1035 + 1036 + ttm_bo_lru_cursor_cleanup_bo(curs); 1037 + if (ret && ret != -EALREADY) 1038 + return ERR_PTR(ret); 1039 + 1040 + spin_lock(lru_lock); 942 1041 } 943 1042 944 - curs->bo = bo; 945 - return bo; 1043 + spin_unlock(lru_lock); 1044 + return res ? bo : NULL; 946 1045 } 947 1046 948 1047 /** ··· 1002 1009 */ 1003 1010 struct ttm_buffer_object *ttm_bo_lru_cursor_next(struct ttm_bo_lru_cursor *curs) 1004 1011 { 1005 - spinlock_t *lru_lock = &curs->res_curs.man->bdev->lru_lock; 1006 - struct ttm_resource *res = NULL; 1007 - struct ttm_buffer_object *bo; 1008 - 1009 - ttm_bo_lru_cursor_cleanup_bo(curs); 1010 - 1011 - spin_lock(lru_lock); 1012 - for (;;) { 1013 - res = ttm_resource_manager_next(&curs->res_curs); 1014 - if (!res) 1015 - break; 1016 - 1017 - bo = ttm_bo_from_res_reserved(res, curs); 1018 - if (bo) 1019 - break; 1020 - } 1021 - 1022 - spin_unlock(lru_lock); 1023 - return res ? bo : NULL; 1012 + return __ttm_bo_lru_cursor_next(curs); 1024 1013 } 1025 1014 EXPORT_SYMBOL(ttm_bo_lru_cursor_next); 1026 1015 ··· 1016 1041 */ 1017 1042 struct ttm_buffer_object *ttm_bo_lru_cursor_first(struct ttm_bo_lru_cursor *curs) 1018 1043 { 1019 - spinlock_t *lru_lock = &curs->res_curs.man->bdev->lru_lock; 1020 - struct ttm_buffer_object *bo; 1021 - struct ttm_resource *res; 1022 - 1023 - spin_lock(lru_lock); 1024 - res = ttm_resource_manager_first(&curs->res_curs); 1025 - if (!res) { 1026 - spin_unlock(lru_lock); 1027 - return NULL; 1028 - } 1029 - 1030 - bo = ttm_bo_from_res_reserved(res, curs); 1031 - spin_unlock(lru_lock); 1032 - 1033 - return bo ? bo : ttm_bo_lru_cursor_next(curs); 1044 + ttm_bo_lru_cursor_cleanup_bo(curs); 1045 + return __ttm_bo_lru_cursor_next(curs); 1034 1046 } 1035 1047 EXPORT_SYMBOL(ttm_bo_lru_cursor_first); 1036 1048

+6 -1

drivers/gpu/drm/xe/xe_shrinker.c

··· 65 65 struct ttm_resource_manager *man = ttm_manager_type(&xe->ttm, mem_type); 66 66 struct ttm_bo_lru_cursor curs; 67 67 struct ttm_buffer_object *ttm_bo; 68 - struct ttm_lru_walk_arg arg = {.ctx = ctx}; 68 + struct ttm_lru_walk_arg arg = { 69 + .ctx = ctx, 70 + .trylock_only = true, 71 + }; 69 72 70 73 if (!man || !man->use_tt) 71 74 continue; ··· 85 82 if (*scanned >= to_scan) 86 83 break; 87 84 } 85 + /* Trylocks should never error, just fail. */ 86 + xe_assert(xe, !IS_ERR(ttm_bo)); 88 87 } 89 88 90 89 return freed;

+7 -2

include/drm/ttm/ttm_bo.h

··· 529 529 * up at looping termination, even if terminated prematurely by, for 530 530 * example a return or break statement. Exiting the loop will also unlock 531 531 * (if needed) and unreference @_bo. 532 + * 533 + * Return: If locking of a bo returns an error, then iteration is terminated 534 + * and @_bo is set to a corresponding error pointer. It's illegal to 535 + * dereference @_bo after loop exit. 532 536 */ 533 537 #define ttm_bo_lru_for_each_reserved_guarded(_cursor, _man, _arg, _bo) \ 534 538 scoped_guard(ttm_bo_lru_cursor, _cursor, _man, _arg) \ 535 - for ((_bo) = ttm_bo_lru_cursor_first(_cursor); (_bo); \ 536 - (_bo) = ttm_bo_lru_cursor_next(_cursor)) 539 + for ((_bo) = ttm_bo_lru_cursor_first(_cursor); \ 540 + !IS_ERR_OR_NULL(_bo); \ 541 + (_bo) = ttm_bo_lru_cursor_next(_cursor)) 537 542 538 543 #endif

Configure Feed

Configure Feed