drm/panthor: Make the timeout per-queue instead of per-job

+201 -80

1 changed file

expand all

drivers

gpu

drm

panthor

panthor_sched.c

+201 -80

drivers/gpu/drm/panthor/panthor_sched.c

··· 364 364 /** @name: DRM scheduler name for this queue. */ 365 365 char *name; 366 366 367 - /** 368 - * @remaining_time: Time remaining before the job timeout expires. 369 - * 370 - * The job timeout is suspended when the queue is not scheduled by the 371 - * FW. Every time we suspend the timer, we need to save the remaining 372 - * time so we can restore it later on. 373 - */ 374 - unsigned long remaining_time; 367 + /** @timeout: Queue timeout related fields. */ 368 + struct { 369 + /** @timeout.work: Work executed when a queue timeout occurs. */ 370 + struct delayed_work work; 375 371 376 - /** @timeout_suspended: True if the job timeout was suspended. */ 377 - bool timeout_suspended; 372 + /** 373 + * @timeout.remaining: Time remaining before a queue timeout. 374 + * 375 + * When the timer is running, this value is set to MAX_SCHEDULE_TIMEOUT. 376 + * When the timer is suspended, it's set to the time remaining when the 377 + * timer was suspended. 378 + */ 379 + unsigned long remaining; 380 + } timeout; 378 381 379 382 /** 380 383 * @doorbell_id: Doorbell assigned to this queue. ··· 902 899 if (IS_ERR_OR_NULL(queue)) 903 900 return; 904 901 902 + /* This should have been disabled before that point. */ 903 + drm_WARN_ON(&group->ptdev->base, 904 + disable_delayed_work_sync(&queue->timeout.work)); 905 + 905 906 if (queue->entity.fence_context) 906 907 drm_sched_entity_destroy(&queue->entity); 907 908 ··· 1053 1046 return 0; 1054 1047 } 1055 1048 1049 + static bool 1050 + group_is_idle(struct panthor_group *group) 1051 + { 1052 + struct panthor_device *ptdev = group->ptdev; 1053 + u32 inactive_queues; 1054 + 1055 + if (group->csg_id >= 0) 1056 + return ptdev->scheduler->csg_slots[group->csg_id].idle; 1057 + 1058 + inactive_queues = group->idle_queues | group->blocked_queues; 1059 + return hweight32(inactive_queues) == group->queue_count; 1060 + } 1061 + 1062 + static void 1063 + queue_reset_timeout_locked(struct panthor_queue *queue) 1064 + { 1065 + lockdep_assert_held(&queue->fence_ctx.lock); 1066 + 1067 + if (queue->timeout.remaining != MAX_SCHEDULE_TIMEOUT) { 1068 + mod_delayed_work(queue->scheduler.timeout_wq, 1069 + &queue->timeout.work, 1070 + msecs_to_jiffies(JOB_TIMEOUT_MS)); 1071 + } 1072 + } 1073 + 1074 + static bool 1075 + group_can_run(struct panthor_group *group) 1076 + { 1077 + return group->state != PANTHOR_CS_GROUP_TERMINATED && 1078 + group->state != PANTHOR_CS_GROUP_UNKNOWN_STATE && 1079 + !group->destroyed && group->fatal_queues == 0 && 1080 + !group->timedout; 1081 + } 1082 + 1083 + static bool 1084 + queue_timeout_is_suspended(struct panthor_queue *queue) 1085 + { 1086 + /* When running, the remaining time is set to MAX_SCHEDULE_TIMEOUT. */ 1087 + return queue->timeout.remaining != MAX_SCHEDULE_TIMEOUT; 1088 + } 1089 + 1090 + static void 1091 + queue_suspend_timeout_locked(struct panthor_queue *queue) 1092 + { 1093 + unsigned long qtimeout, now; 1094 + struct panthor_group *group; 1095 + struct panthor_job *job; 1096 + bool timer_was_active; 1097 + 1098 + lockdep_assert_held(&queue->fence_ctx.lock); 1099 + 1100 + /* Already suspended, nothing to do. */ 1101 + if (queue_timeout_is_suspended(queue)) 1102 + return; 1103 + 1104 + job = list_first_entry_or_null(&queue->fence_ctx.in_flight_jobs, 1105 + struct panthor_job, node); 1106 + group = job ? job->group : NULL; 1107 + 1108 + /* If the queue is blocked and the group is idle, we want the timer to 1109 + * keep running because the group can't be unblocked by other queues, 1110 + * so it has to come from an external source, and we want to timebox 1111 + * this external signalling. 1112 + */ 1113 + if (group && group_can_run(group) && 1114 + (group->blocked_queues & BIT(job->queue_idx)) && 1115 + group_is_idle(group)) 1116 + return; 1117 + 1118 + now = jiffies; 1119 + qtimeout = queue->timeout.work.timer.expires; 1120 + 1121 + /* Cancel the timer. */ 1122 + timer_was_active = cancel_delayed_work(&queue->timeout.work); 1123 + if (!timer_was_active || !job) 1124 + queue->timeout.remaining = msecs_to_jiffies(JOB_TIMEOUT_MS); 1125 + else if (time_after(qtimeout, now)) 1126 + queue->timeout.remaining = qtimeout - now; 1127 + else 1128 + queue->timeout.remaining = 0; 1129 + 1130 + if (WARN_ON_ONCE(queue->timeout.remaining > msecs_to_jiffies(JOB_TIMEOUT_MS))) 1131 + queue->timeout.remaining = msecs_to_jiffies(JOB_TIMEOUT_MS); 1132 + } 1133 + 1134 + static void 1135 + queue_suspend_timeout(struct panthor_queue *queue) 1136 + { 1137 + spin_lock(&queue->fence_ctx.lock); 1138 + queue_suspend_timeout_locked(queue); 1139 + spin_unlock(&queue->fence_ctx.lock); 1140 + } 1141 + 1142 + static void 1143 + queue_resume_timeout(struct panthor_queue *queue) 1144 + { 1145 + spin_lock(&queue->fence_ctx.lock); 1146 + 1147 + if (queue_timeout_is_suspended(queue)) { 1148 + mod_delayed_work(queue->scheduler.timeout_wq, 1149 + &queue->timeout.work, 1150 + queue->timeout.remaining); 1151 + 1152 + queue->timeout.remaining = MAX_SCHEDULE_TIMEOUT; 1153 + } 1154 + 1155 + spin_unlock(&queue->fence_ctx.lock); 1156 + } 1157 + 1056 1158 /** 1057 1159 * cs_slot_prog_locked() - Program a queue slot 1058 1160 * @ptdev: Device. ··· 1200 1084 CS_IDLE_EMPTY | 1201 1085 CS_STATE_MASK | 1202 1086 CS_EXTRACT_EVENT); 1203 - if (queue->iface.input->insert != queue->iface.input->extract && queue->timeout_suspended) { 1204 - drm_sched_resume_timeout(&queue->scheduler, queue->remaining_time); 1205 - queue->timeout_suspended = false; 1206 - } 1087 + if (queue->iface.input->insert != queue->iface.input->extract) 1088 + queue_resume_timeout(queue); 1207 1089 } 1208 1090 1209 1091 /** ··· 1228 1114 CS_STATE_STOP, 1229 1115 CS_STATE_MASK); 1230 1116 1231 - /* If the queue is blocked, we want to keep the timeout running, so 1232 - * we can detect unbounded waits and kill the group when that happens. 1233 - */ 1234 - if (!(group->blocked_queues & BIT(cs_id)) && !queue->timeout_suspended) { 1235 - queue->remaining_time = drm_sched_suspend_timeout(&queue->scheduler); 1236 - queue->timeout_suspended = true; 1237 - WARN_ON(queue->remaining_time > msecs_to_jiffies(JOB_TIMEOUT_MS)); 1238 - } 1117 + queue_suspend_timeout(queue); 1239 1118 1240 1119 return 0; 1241 1120 } ··· 2028 1921 return ctx->group_count == sched->csg_slot_count; 2029 1922 } 2030 1923 2031 - static bool 2032 - group_is_idle(struct panthor_group *group) 2033 - { 2034 - struct panthor_device *ptdev = group->ptdev; 2035 - u32 inactive_queues; 2036 - 2037 - if (group->csg_id >= 0) 2038 - return ptdev->scheduler->csg_slots[group->csg_id].idle; 2039 - 2040 - inactive_queues = group->idle_queues | group->blocked_queues; 2041 - return hweight32(inactive_queues) == group->queue_count; 2042 - } 2043 - 2044 - static bool 2045 - group_can_run(struct panthor_group *group) 2046 - { 2047 - return group->state != PANTHOR_CS_GROUP_TERMINATED && 2048 - group->state != PANTHOR_CS_GROUP_UNKNOWN_STATE && 2049 - !group->destroyed && group->fatal_queues == 0 && 2050 - !group->timedout; 2051 - } 2052 - 2053 1924 static void 2054 1925 tick_ctx_pick_groups_from_list(const struct panthor_scheduler *sched, 2055 1926 struct panthor_sched_tick_ctx *ctx, ··· 2709 2624 static void queue_stop(struct panthor_queue *queue, 2710 2625 struct panthor_job *bad_job) 2711 2626 { 2627 + disable_delayed_work_sync(&queue->timeout.work); 2712 2628 drm_sched_stop(&queue->scheduler, bad_job ? &bad_job->base : NULL); 2713 2629 } 2714 2630 ··· 2721 2635 list_for_each_entry(job, &queue->scheduler.pending_list, base.list) 2722 2636 job->base.s_fence->parent = dma_fence_get(job->done_fence); 2723 2637 2638 + enable_delayed_work(&queue->timeout.work); 2724 2639 drm_sched_start(&queue->scheduler, 0); 2725 2640 } 2726 2641 ··· 2788 2701 { 2789 2702 struct panthor_scheduler *sched = ptdev->scheduler; 2790 2703 struct panthor_csg_slots_upd_ctx upd_ctx; 2791 - struct panthor_group *group; 2792 2704 u32 suspended_slots; 2793 2705 u32 i; 2794 2706 ··· 2877 2791 2878 2792 for (i = 0; i < sched->csg_slot_count; i++) { 2879 2793 struct panthor_csg_slot *csg_slot = &sched->csg_slots[i]; 2794 + struct panthor_group *group = csg_slot->group; 2880 2795 2881 - group = csg_slot->group; 2882 2796 if (!group) 2883 2797 continue; 2884 2798 ··· 3007 2921 xa_unlock(&gpool->xa); 3008 2922 } 3009 2923 3010 - static void group_sync_upd_work(struct work_struct *work) 2924 + static bool queue_check_job_completion(struct panthor_queue *queue) 3011 2925 { 3012 - struct panthor_group *group = 3013 - container_of(work, struct panthor_group, sync_upd_work); 2926 + struct panthor_syncobj_64b *syncobj = NULL; 3014 2927 struct panthor_job *job, *job_tmp; 2928 + bool cookie, progress = false; 3015 2929 LIST_HEAD(done_jobs); 3016 - u32 queue_idx; 3017 - bool cookie; 3018 2930 3019 2931 cookie = dma_fence_begin_signalling(); 3020 - for (queue_idx = 0; queue_idx < group->queue_count; queue_idx++) { 3021 - struct panthor_queue *queue = group->queues[queue_idx]; 3022 - struct panthor_syncobj_64b *syncobj; 2932 + spin_lock(&queue->fence_ctx.lock); 2933 + list_for_each_entry_safe(job, job_tmp, &queue->fence_ctx.in_flight_jobs, node) { 2934 + if (!syncobj) { 2935 + struct panthor_group *group = job->group; 3023 2936 3024 - if (!queue) 3025 - continue; 3026 - 3027 - syncobj = group->syncobjs->kmap + (queue_idx * sizeof(*syncobj)); 3028 - 3029 - spin_lock(&queue->fence_ctx.lock); 3030 - list_for_each_entry_safe(job, job_tmp, &queue->fence_ctx.in_flight_jobs, node) { 3031 - if (syncobj->seqno < job->done_fence->seqno) 3032 - break; 3033 - 3034 - list_move_tail(&job->node, &done_jobs); 3035 - dma_fence_signal_locked(job->done_fence); 2937 + syncobj = group->syncobjs->kmap + 2938 + (job->queue_idx * sizeof(*syncobj)); 3036 2939 } 3037 - spin_unlock(&queue->fence_ctx.lock); 2940 + 2941 + if (syncobj->seqno < job->done_fence->seqno) 2942 + break; 2943 + 2944 + list_move_tail(&job->node, &done_jobs); 2945 + dma_fence_signal_locked(job->done_fence); 3038 2946 } 2947 + 2948 + if (list_empty(&queue->fence_ctx.in_flight_jobs)) { 2949 + /* If we have no job left, we cancel the timer, and reset remaining 2950 + * time to its default so it can be restarted next time 2951 + * queue_resume_timeout() is called. 2952 + */ 2953 + queue_suspend_timeout_locked(queue); 2954 + 2955 + /* If there's no job pending, we consider it progress to avoid a 2956 + * spurious timeout if the timeout handler and the sync update 2957 + * handler raced. 2958 + */ 2959 + progress = true; 2960 + } else if (!list_empty(&done_jobs)) { 2961 + queue_reset_timeout_locked(queue); 2962 + progress = true; 2963 + } 2964 + spin_unlock(&queue->fence_ctx.lock); 3039 2965 dma_fence_end_signalling(cookie); 3040 2966 3041 2967 list_for_each_entry_safe(job, job_tmp, &done_jobs, node) { ··· 3056 2958 list_del_init(&job->node); 3057 2959 panthor_job_put(&job->base); 3058 2960 } 2961 + 2962 + return progress; 2963 + } 2964 + 2965 + static void group_sync_upd_work(struct work_struct *work) 2966 + { 2967 + struct panthor_group *group = 2968 + container_of(work, struct panthor_group, sync_upd_work); 2969 + u32 queue_idx; 2970 + bool cookie; 2971 + 2972 + cookie = dma_fence_begin_signalling(); 2973 + for (queue_idx = 0; queue_idx < group->queue_count; queue_idx++) { 2974 + struct panthor_queue *queue = group->queues[queue_idx]; 2975 + 2976 + if (!queue) 2977 + continue; 2978 + 2979 + queue_check_job_completion(queue); 2980 + } 2981 + dma_fence_end_signalling(cookie); 3059 2982 3060 2983 group_put(group); 3061 2984 } ··· 3325 3206 queue->iface.input->insert = job->ringbuf.end; 3326 3207 3327 3208 if (group->csg_id < 0) { 3328 - /* If the queue is blocked, we want to keep the timeout running, so we 3329 - * can detect unbounded waits and kill the group when that happens. 3330 - * Otherwise, we suspend the timeout so the time we spend waiting for 3331 - * a CSG slot is not counted. 3332 - */ 3333 - if (!(group->blocked_queues & BIT(job->queue_idx)) && 3334 - !queue->timeout_suspended) { 3335 - queue->remaining_time = drm_sched_suspend_timeout(&queue->scheduler); 3336 - queue->timeout_suspended = true; 3337 - } 3338 - 3339 3209 group_schedule_locked(group, BIT(job->queue_idx)); 3340 3210 } else { 3341 3211 gpu_write(ptdev, CSF_DOORBELL(queue->doorbell_id), 1); ··· 3333 3225 pm_runtime_get(ptdev->base.dev); 3334 3226 sched->pm.has_ref = true; 3335 3227 } 3228 + queue_resume_timeout(queue); 3336 3229 panthor_devfreq_record_busy(sched->ptdev); 3337 3230 } 3338 3231 ··· 3383 3274 mutex_unlock(&sched->lock); 3384 3275 3385 3276 queue_start(queue); 3386 - 3387 3277 return DRM_GPU_SCHED_STAT_RESET; 3388 3278 } 3389 3279 ··· 3425 3317 return DIV_ROUND_UP(cs_ringbuf_size, min_profiled_job_instrs * sizeof(u64)); 3426 3318 } 3427 3319 3320 + static void queue_timeout_work(struct work_struct *work) 3321 + { 3322 + struct panthor_queue *queue = container_of(work, struct panthor_queue, 3323 + timeout.work.work); 3324 + bool progress; 3325 + 3326 + progress = queue_check_job_completion(queue); 3327 + if (!progress) 3328 + drm_sched_fault(&queue->scheduler); 3329 + } 3330 + 3428 3331 static struct panthor_queue * 3429 3332 group_create_queue(struct panthor_group *group, 3430 3333 const struct drm_panthor_queue_create *args, ··· 3452 3333 * their profiling status. 3453 3334 */ 3454 3335 .credit_limit = args->ringbuf_size / sizeof(u64), 3455 - .timeout = msecs_to_jiffies(JOB_TIMEOUT_MS), 3336 + .timeout = MAX_SCHEDULE_TIMEOUT, 3456 3337 .timeout_wq = group->ptdev->reset.wq, 3457 3338 .dev = group->ptdev->base.dev, 3458 3339 }; ··· 3474 3355 if (!queue) 3475 3356 return ERR_PTR(-ENOMEM); 3476 3357 3358 + queue->timeout.remaining = msecs_to_jiffies(JOB_TIMEOUT_MS); 3359 + INIT_DELAYED_WORK(&queue->timeout.work, queue_timeout_work); 3477 3360 queue->fence_ctx.id = dma_fence_context_alloc(1); 3478 3361 spin_lock_init(&queue->fence_ctx.lock); 3479 3362 INIT_LIST_HEAD(&queue->fence_ctx.in_flight_jobs);

Configure Feed

Configure Feed