Merge tag 'drm-next-2025-12-05' of https://gitlab.freedesktop.org/drm/kernel

+7

MAINTAINERS

··· 27221 27221 S: Maintained 27222 27222 F: drivers/vfio/pci/virtio 27223 27223 27224 + VFIO XE PCI DRIVER 27225 + M: Michał Winiarski <michal.winiarski@intel.com> 27226 + L: kvm@vger.kernel.org 27227 + L: intel-xe@lists.freedesktop.org 27228 + S: Supported 27229 + F: drivers/vfio/pci/xe 27230 + 27224 27231 VGA_SWITCHEROO 27225 27232 R: Lukas Wunner <lukas@wunner.de> 27226 27233 S: Maintained

+2

drivers/gpu/drm/i915/Makefile

··· 239 239 display/intel_cdclk.o \ 240 240 display/intel_cmtg.o \ 241 241 display/intel_color.o \ 242 + display/intel_colorop.o \ 243 + display/intel_color_pipeline.o \ 242 244 display/intel_combo_phy.o \ 243 245 display/intel_connector.o \ 244 246 display/intel_crtc.o \

+335

drivers/gpu/drm/i915/display/intel_color.c

··· 32 32 #include "intel_display_utils.h" 33 33 #include "intel_dsb.h" 34 34 #include "intel_vrr.h" 35 + #include "skl_universal_plane.h" 36 + #include "skl_universal_plane_regs.h" 35 37 36 38 struct intel_color_funcs { 37 39 int (*color_check)(struct intel_atomic_state *state, ··· 89 87 * Read config other than LUTs and CSCs, before them. Optional. 90 88 */ 91 89 void (*get_config)(struct intel_crtc_state *crtc_state); 90 + 91 + /* Plane CSC*/ 92 + void (*load_plane_csc_matrix)(struct intel_dsb *dsb, 93 + const struct intel_plane_state *plane_state); 94 + 95 + /* Plane Pre/Post CSC */ 96 + void (*load_plane_luts)(struct intel_dsb *dsb, 97 + const struct intel_plane_state *plane_state); 92 98 }; 93 99 94 100 #define CTM_COEFF_SIGN (1ULL << 63) ··· 618 608 619 609 if (CTM_COEFF_NEGATIVE(coeff)) 620 610 c = -c; 611 + 612 + int_bits = max(int_bits, 1); 621 613 622 614 c = clamp(c, -(s64)BIT(int_bits + frac_bits - 1), 623 615 (s64)(BIT(int_bits + frac_bits - 1) - 1)); ··· 3848 3836 } 3849 3837 } 3850 3838 3839 + static void 3840 + xelpd_load_plane_csc_matrix(struct intel_dsb *dsb, 3841 + const struct intel_plane_state *plane_state) 3842 + { 3843 + struct intel_display *display = to_intel_display(plane_state); 3844 + const struct drm_plane_state *state = &plane_state->uapi; 3845 + enum pipe pipe = to_intel_plane(state->plane)->pipe; 3846 + enum plane_id plane = to_intel_plane(state->plane)->id; 3847 + const struct drm_property_blob *blob = plane_state->hw.ctm; 3848 + struct drm_color_ctm_3x4 *ctm; 3849 + const u64 *input; 3850 + u16 coeffs[9] = {}; 3851 + int i, j; 3852 + 3853 + if (!icl_is_hdr_plane(display, plane) || !blob) 3854 + return; 3855 + 3856 + ctm = blob->data; 3857 + input = ctm->matrix; 3858 + 3859 + /* 3860 + * Convert fixed point S31.32 input to format supported by the 3861 + * hardware. 3862 + */ 3863 + for (i = 0, j = 0; i < ARRAY_SIZE(coeffs); i++) { 3864 + u64 abs_coeff = ((1ULL << 63) - 1) & input[j]; 3865 + 3866 + /* 3867 + * Clamp input value to min/max supported by 3868 + * hardware. 3869 + */ 3870 + abs_coeff = clamp_val(abs_coeff, 0, CTM_COEFF_4_0 - 1); 3871 + 3872 + /* sign bit */ 3873 + if (CTM_COEFF_NEGATIVE(input[j])) 3874 + coeffs[i] |= 1 << 15; 3875 + 3876 + if (abs_coeff < CTM_COEFF_0_125) 3877 + coeffs[i] |= (3 << 12) | 3878 + ILK_CSC_COEFF_FP(abs_coeff, 12); 3879 + else if (abs_coeff < CTM_COEFF_0_25) 3880 + coeffs[i] |= (2 << 12) | 3881 + ILK_CSC_COEFF_FP(abs_coeff, 11); 3882 + else if (abs_coeff < CTM_COEFF_0_5) 3883 + coeffs[i] |= (1 << 12) | 3884 + ILK_CSC_COEFF_FP(abs_coeff, 10); 3885 + else if (abs_coeff < CTM_COEFF_1_0) 3886 + coeffs[i] |= ILK_CSC_COEFF_FP(abs_coeff, 9); 3887 + else if (abs_coeff < CTM_COEFF_2_0) 3888 + coeffs[i] |= (7 << 12) | 3889 + ILK_CSC_COEFF_FP(abs_coeff, 8); 3890 + else 3891 + coeffs[i] |= (6 << 12) | 3892 + ILK_CSC_COEFF_FP(abs_coeff, 7); 3893 + 3894 + /* Skip postoffs */ 3895 + if (!((j + 2) % 4)) 3896 + j += 2; 3897 + else 3898 + j++; 3899 + } 3900 + 3901 + intel_de_write_dsb(display, dsb, PLANE_CSC_COEFF(pipe, plane, 0), 3902 + coeffs[0] << 16 | coeffs[1]); 3903 + intel_de_write_dsb(display, dsb, PLANE_CSC_COEFF(pipe, plane, 1), 3904 + coeffs[2] << 16); 3905 + 3906 + intel_de_write_dsb(display, dsb, PLANE_CSC_COEFF(pipe, plane, 2), 3907 + coeffs[3] << 16 | coeffs[4]); 3908 + intel_de_write_dsb(display, dsb, PLANE_CSC_COEFF(pipe, plane, 3), 3909 + coeffs[5] << 16); 3910 + 3911 + intel_de_write_dsb(display, dsb, PLANE_CSC_COEFF(pipe, plane, 4), 3912 + coeffs[6] << 16 | coeffs[7]); 3913 + intel_de_write_dsb(display, dsb, PLANE_CSC_COEFF(pipe, plane, 5), 3914 + coeffs[8] << 16); 3915 + 3916 + intel_de_write_dsb(display, dsb, PLANE_CSC_PREOFF(pipe, plane, 0), 0); 3917 + intel_de_write_dsb(display, dsb, PLANE_CSC_PREOFF(pipe, plane, 1), 0); 3918 + intel_de_write_dsb(display, dsb, PLANE_CSC_PREOFF(pipe, plane, 2), 0); 3919 + 3920 + /* 3921 + * Conversion from S31.32 to S0.12. BIT[12] is the signed bit 3922 + */ 3923 + intel_de_write_dsb(display, dsb, 3924 + PLANE_CSC_POSTOFF(pipe, plane, 0), 3925 + ctm_to_twos_complement(input[3], 0, 12)); 3926 + intel_de_write_dsb(display, dsb, 3927 + PLANE_CSC_POSTOFF(pipe, plane, 1), 3928 + ctm_to_twos_complement(input[7], 0, 12)); 3929 + intel_de_write_dsb(display, dsb, 3930 + PLANE_CSC_POSTOFF(pipe, plane, 2), 3931 + ctm_to_twos_complement(input[11], 0, 12)); 3932 + } 3933 + 3934 + static void 3935 + xelpd_program_plane_pre_csc_lut(struct intel_dsb *dsb, 3936 + const struct intel_plane_state *plane_state) 3937 + { 3938 + struct intel_display *display = to_intel_display(plane_state); 3939 + const struct drm_plane_state *state = &plane_state->uapi; 3940 + enum pipe pipe = to_intel_plane(state->plane)->pipe; 3941 + enum plane_id plane = to_intel_plane(state->plane)->id; 3942 + const struct drm_color_lut32 *pre_csc_lut = plane_state->hw.degamma_lut->data; 3943 + u32 i, lut_size; 3944 + 3945 + if (icl_is_hdr_plane(display, plane)) { 3946 + lut_size = 128; 3947 + 3948 + intel_de_write_dsb(display, dsb, 3949 + PLANE_PRE_CSC_GAMC_INDEX_ENH(pipe, plane, 0), 3950 + PLANE_PAL_PREC_AUTO_INCREMENT); 3951 + 3952 + if (pre_csc_lut) { 3953 + for (i = 0; i < lut_size; i++) { 3954 + u32 lut_val = drm_color_lut32_extract(pre_csc_lut[i].green, 24); 3955 + 3956 + intel_de_write_dsb(display, dsb, 3957 + PLANE_PRE_CSC_GAMC_DATA_ENH(pipe, plane, 0), 3958 + lut_val); 3959 + } 3960 + 3961 + /* Program the max register to clamp values > 1.0. */ 3962 + /* TODO: Restrict to 0x7ffffff */ 3963 + do { 3964 + intel_de_write_dsb(display, dsb, 3965 + PLANE_PRE_CSC_GAMC_DATA_ENH(pipe, plane, 0), 3966 + (1 << 24)); 3967 + } while (i++ > 130); 3968 + } else { 3969 + for (i = 0; i < lut_size; i++) { 3970 + u32 v = (i * ((1 << 24) - 1)) / (lut_size - 1); 3971 + 3972 + intel_de_write_dsb(display, dsb, 3973 + PLANE_PRE_CSC_GAMC_DATA_ENH(pipe, plane, 0), v); 3974 + } 3975 + 3976 + do { 3977 + intel_de_write_dsb(display, dsb, 3978 + PLANE_PRE_CSC_GAMC_DATA_ENH(pipe, plane, 0), 3979 + 1 << 24); 3980 + } while (i++ < 130); 3981 + } 3982 + 3983 + intel_de_write_dsb(display, dsb, PLANE_PRE_CSC_GAMC_INDEX_ENH(pipe, plane, 0), 0); 3984 + } 3985 + } 3986 + 3987 + static void 3988 + xelpd_program_plane_post_csc_lut(struct intel_dsb *dsb, 3989 + const struct intel_plane_state *plane_state) 3990 + { 3991 + struct intel_display *display = to_intel_display(plane_state); 3992 + const struct drm_plane_state *state = &plane_state->uapi; 3993 + enum pipe pipe = to_intel_plane(state->plane)->pipe; 3994 + enum plane_id plane = to_intel_plane(state->plane)->id; 3995 + const struct drm_color_lut32 *post_csc_lut = plane_state->hw.gamma_lut->data; 3996 + u32 i, lut_size, lut_val; 3997 + 3998 + if (icl_is_hdr_plane(display, plane)) { 3999 + intel_de_write_dsb(display, dsb, PLANE_POST_CSC_GAMC_INDEX_ENH(pipe, plane, 0), 4000 + PLANE_PAL_PREC_AUTO_INCREMENT); 4001 + /* TODO: Add macro */ 4002 + intel_de_write_dsb(display, dsb, PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH(pipe, plane, 0), 4003 + PLANE_PAL_PREC_AUTO_INCREMENT); 4004 + if (post_csc_lut) { 4005 + lut_size = 32; 4006 + for (i = 0; i < lut_size; i++) { 4007 + lut_val = drm_color_lut32_extract(post_csc_lut[i].green, 24); 4008 + 4009 + intel_de_write_dsb(display, dsb, 4010 + PLANE_POST_CSC_GAMC_DATA_ENH(pipe, plane, 0), 4011 + lut_val); 4012 + } 4013 + 4014 + /* Segment 2 */ 4015 + do { 4016 + intel_de_write_dsb(display, dsb, 4017 + PLANE_POST_CSC_GAMC_DATA_ENH(pipe, plane, 0), 4018 + (1 << 24)); 4019 + } while (i++ < 34); 4020 + } else { 4021 + /*TODO: Add for segment 0 */ 4022 + lut_size = 32; 4023 + for (i = 0; i < lut_size; i++) { 4024 + u32 v = (i * ((1 << 24) - 1)) / (lut_size - 1); 4025 + 4026 + intel_de_write_dsb(display, dsb, 4027 + PLANE_POST_CSC_GAMC_DATA_ENH(pipe, plane, 0), v); 4028 + } 4029 + 4030 + do { 4031 + intel_de_write_dsb(display, dsb, 4032 + PLANE_POST_CSC_GAMC_DATA_ENH(pipe, plane, 0), 4033 + 1 << 24); 4034 + } while (i++ < 34); 4035 + } 4036 + 4037 + intel_de_write_dsb(display, dsb, PLANE_POST_CSC_GAMC_INDEX_ENH(pipe, plane, 0), 0); 4038 + intel_de_write_dsb(display, dsb, 4039 + PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH(pipe, plane, 0), 0); 4040 + } 4041 + } 4042 + 4043 + static void 4044 + xelpd_plane_load_luts(struct intel_dsb *dsb, const struct intel_plane_state *plane_state) 4045 + { 4046 + if (plane_state->hw.degamma_lut) 4047 + xelpd_program_plane_pre_csc_lut(dsb, plane_state); 4048 + 4049 + if (plane_state->hw.gamma_lut) 4050 + xelpd_program_plane_post_csc_lut(dsb, plane_state); 4051 + } 4052 + 4053 + static u32 glk_3dlut_10(const struct drm_color_lut32 *color) 4054 + { 4055 + return REG_FIELD_PREP(LUT_3D_DATA_RED_MASK, drm_color_lut32_extract(color->red, 10)) | 4056 + REG_FIELD_PREP(LUT_3D_DATA_GREEN_MASK, drm_color_lut32_extract(color->green, 10)) | 4057 + REG_FIELD_PREP(LUT_3D_DATA_BLUE_MASK, drm_color_lut32_extract(color->blue, 10)); 4058 + } 4059 + 4060 + static void glk_load_lut_3d(struct intel_dsb *dsb, 4061 + struct intel_crtc *crtc, 4062 + const struct drm_property_blob *blob) 4063 + { 4064 + struct intel_display *display = to_intel_display(crtc->base.dev); 4065 + const struct drm_color_lut32 *lut = blob->data; 4066 + int i, lut_size = drm_color_lut32_size(blob); 4067 + enum pipe pipe = crtc->pipe; 4068 + 4069 + if (!dsb && intel_de_read(display, LUT_3D_CTL(pipe)) & LUT_3D_READY) { 4070 + drm_err(display->drm, "[CRTC:%d:%s] 3D LUT not ready, not loading LUTs\n", 4071 + crtc->base.base.id, crtc->base.name); 4072 + return; 4073 + } 4074 + 4075 + intel_de_write_dsb(display, dsb, LUT_3D_INDEX(pipe), LUT_3D_AUTO_INCREMENT); 4076 + for (i = 0; i < lut_size; i++) 4077 + intel_de_write_dsb(display, dsb, LUT_3D_DATA(pipe), glk_3dlut_10(&lut[i])); 4078 + intel_de_write_dsb(display, dsb, LUT_3D_INDEX(pipe), 0); 4079 + } 4080 + 4081 + static void glk_lut_3d_commit(struct intel_dsb *dsb, struct intel_crtc *crtc, bool enable) 4082 + { 4083 + struct intel_display *display = to_intel_display(crtc); 4084 + enum pipe pipe = crtc->pipe; 4085 + u32 val = 0; 4086 + 4087 + if (!dsb && intel_de_read(display, LUT_3D_CTL(pipe)) & LUT_3D_READY) { 4088 + drm_err(display->drm, "[CRTC:%d:%s] 3D LUT not ready, not committing change\n", 4089 + crtc->base.base.id, crtc->base.name); 4090 + return; 4091 + } 4092 + 4093 + if (enable) 4094 + val = LUT_3D_ENABLE | LUT_3D_READY | LUT_3D_BIND_PLANE_1; 4095 + 4096 + intel_de_write_dsb(display, dsb, LUT_3D_CTL(pipe), val); 4097 + } 4098 + 3851 4099 static const struct intel_color_funcs chv_color_funcs = { 3852 4100 .color_check = chv_color_check, 3853 4101 .color_commit_arm = i9xx_color_commit_arm, ··· 4155 3883 .lut_equal = icl_lut_equal, 4156 3884 .read_csc = icl_read_csc, 4157 3885 .get_config = skl_get_config, 3886 + .load_plane_csc_matrix = xelpd_load_plane_csc_matrix, 3887 + .load_plane_luts = xelpd_plane_load_luts, 4158 3888 }; 4159 3889 4160 3890 static const struct intel_color_funcs icl_color_funcs = { ··· 4236 3962 .read_csc = ilk_read_csc, 4237 3963 .get_config = ilk_get_config, 4238 3964 }; 3965 + 3966 + void intel_color_plane_commit_arm(struct intel_dsb *dsb, 3967 + const struct intel_plane_state *plane_state) 3968 + { 3969 + struct intel_display *display = to_intel_display(plane_state); 3970 + struct intel_crtc *crtc = to_intel_crtc(plane_state->uapi.crtc); 3971 + 3972 + if (crtc && intel_color_crtc_has_3dlut(display, crtc->pipe)) 3973 + glk_lut_3d_commit(dsb, crtc, !!plane_state->hw.lut_3d); 3974 + } 3975 + 3976 + static void 3977 + intel_color_load_plane_csc_matrix(struct intel_dsb *dsb, 3978 + const struct intel_plane_state *plane_state) 3979 + { 3980 + struct intel_display *display = to_intel_display(plane_state); 3981 + 3982 + if (display->funcs.color->load_plane_csc_matrix) 3983 + display->funcs.color->load_plane_csc_matrix(dsb, plane_state); 3984 + } 3985 + 3986 + static void 3987 + intel_color_load_plane_luts(struct intel_dsb *dsb, 3988 + const struct intel_plane_state *plane_state) 3989 + { 3990 + struct intel_display *display = to_intel_display(plane_state); 3991 + 3992 + if (display->funcs.color->load_plane_luts) 3993 + display->funcs.color->load_plane_luts(dsb, plane_state); 3994 + } 3995 + 3996 + bool 3997 + intel_color_crtc_has_3dlut(struct intel_display *display, enum pipe pipe) 3998 + { 3999 + if (DISPLAY_VER(display) >= 12) 4000 + return pipe == PIPE_A || pipe == PIPE_B; 4001 + else 4002 + return false; 4003 + } 4004 + 4005 + static void 4006 + intel_color_load_3dlut(struct intel_dsb *dsb, 4007 + const struct intel_plane_state *plane_state) 4008 + { 4009 + struct intel_display *display = to_intel_display(plane_state); 4010 + struct intel_crtc *crtc = to_intel_crtc(plane_state->uapi.crtc); 4011 + 4012 + if (crtc && intel_color_crtc_has_3dlut(display, crtc->pipe)) 4013 + glk_load_lut_3d(dsb, crtc, plane_state->hw.lut_3d); 4014 + } 4015 + 4016 + void intel_color_plane_program_pipeline(struct intel_dsb *dsb, 4017 + const struct intel_plane_state *plane_state) 4018 + { 4019 + if (plane_state->hw.ctm) 4020 + intel_color_load_plane_csc_matrix(dsb, plane_state); 4021 + if (plane_state->hw.degamma_lut || plane_state->hw.gamma_lut) 4022 + intel_color_load_plane_luts(dsb, plane_state); 4023 + if (plane_state->hw.lut_3d) 4024 + intel_color_load_3dlut(dsb, plane_state); 4025 + } 4239 4026 4240 4027 void intel_color_crtc_init(struct intel_crtc *crtc) 4241 4028 {

+7 -1

drivers/gpu/drm/i915/display/intel_color.h

··· 13 13 struct intel_crtc; 14 14 struct intel_display; 15 15 struct intel_dsb; 16 + struct intel_plane_state; 16 17 struct drm_property_blob; 18 + enum pipe; 17 19 18 20 void intel_color_init_hooks(struct intel_display *display); 19 21 int intel_color_init(struct intel_display *display); ··· 42 40 const struct drm_property_blob *blob2, 43 41 bool is_pre_csc_lut); 44 42 void intel_color_assert_luts(const struct intel_crtc_state *crtc_state); 45 - 43 + void intel_color_plane_program_pipeline(struct intel_dsb *dsb, 44 + const struct intel_plane_state *plane_state); 45 + void intel_color_plane_commit_arm(struct intel_dsb *dsb, 46 + const struct intel_plane_state *plane_state); 47 + bool intel_color_crtc_has_3dlut(struct intel_display *display, enum pipe pipe); 46 48 #endif /* __INTEL_COLOR_H__ */

+99

drivers/gpu/drm/i915/display/intel_color_pipeline.c

··· 1 + // SPDX-License-Identifier: MIT 2 + /* 3 + * Copyright © 2025 Intel Corporation 4 + */ 5 + #include "intel_color.h" 6 + #include "intel_colorop.h" 7 + #include "intel_color_pipeline.h" 8 + #include "intel_de.h" 9 + #include "intel_display_types.h" 10 + #include "skl_universal_plane.h" 11 + 12 + #define MAX_COLOR_PIPELINES 1 13 + #define PLANE_DEGAMMA_SIZE 128 14 + #define PLANE_GAMMA_SIZE 32 15 + 16 + static 17 + int _intel_color_pipeline_plane_init(struct drm_plane *plane, struct drm_prop_enum_list *list, 18 + enum pipe pipe) 19 + { 20 + struct drm_device *dev = plane->dev; 21 + struct intel_display *display = to_intel_display(dev); 22 + struct drm_colorop *prev_op; 23 + struct intel_colorop *colorop; 24 + int ret; 25 + 26 + colorop = intel_colorop_create(INTEL_PLANE_CB_PRE_CSC_LUT); 27 + 28 + ret = drm_plane_colorop_curve_1d_lut_init(dev, &colorop->base, plane, 29 + PLANE_DEGAMMA_SIZE, 30 + DRM_COLOROP_LUT1D_INTERPOLATION_LINEAR, 31 + DRM_COLOROP_FLAG_ALLOW_BYPASS); 32 + 33 + if (ret) 34 + return ret; 35 + 36 + list->type = colorop->base.base.id; 37 + list->name = kasprintf(GFP_KERNEL, "Color Pipeline %d", colorop->base.base.id); 38 + 39 + /* TODO: handle failures and clean up */ 40 + prev_op = &colorop->base; 41 + 42 + if (DISPLAY_VER(display) >= 35 && 43 + intel_color_crtc_has_3dlut(display, pipe) && 44 + plane->type == DRM_PLANE_TYPE_PRIMARY) { 45 + colorop = intel_colorop_create(INTEL_PLANE_CB_3DLUT); 46 + 47 + ret = drm_plane_colorop_3dlut_init(dev, &colorop->base, plane, 17, 48 + DRM_COLOROP_LUT3D_INTERPOLATION_TETRAHEDRAL, 49 + true); 50 + if (ret) 51 + return ret; 52 + 53 + drm_colorop_set_next_property(prev_op, &colorop->base); 54 + 55 + prev_op = &colorop->base; 56 + } 57 + 58 + colorop = intel_colorop_create(INTEL_PLANE_CB_CSC); 59 + ret = drm_plane_colorop_ctm_3x4_init(dev, &colorop->base, plane, 60 + DRM_COLOROP_FLAG_ALLOW_BYPASS); 61 + if (ret) 62 + return ret; 63 + 64 + drm_colorop_set_next_property(prev_op, &colorop->base); 65 + prev_op = &colorop->base; 66 + 67 + colorop = intel_colorop_create(INTEL_PLANE_CB_POST_CSC_LUT); 68 + ret = drm_plane_colorop_curve_1d_lut_init(dev, &colorop->base, plane, 69 + PLANE_GAMMA_SIZE, 70 + DRM_COLOROP_LUT1D_INTERPOLATION_LINEAR, 71 + DRM_COLOROP_FLAG_ALLOW_BYPASS); 72 + if (ret) 73 + return ret; 74 + 75 + drm_colorop_set_next_property(prev_op, &colorop->base); 76 + 77 + return 0; 78 + } 79 + 80 + int intel_color_pipeline_plane_init(struct drm_plane *plane, enum pipe pipe) 81 + { 82 + struct drm_device *dev = plane->dev; 83 + struct intel_display *display = to_intel_display(dev); 84 + struct drm_prop_enum_list pipelines[MAX_COLOR_PIPELINES]; 85 + int len = 0; 86 + int ret; 87 + 88 + /* Currently expose pipeline only for HDR planes */ 89 + if (!icl_is_hdr_plane(display, to_intel_plane(plane)->id)) 90 + return 0; 91 + 92 + /* Add pipeline consisting of transfer functions */ 93 + ret = _intel_color_pipeline_plane_init(plane, &pipelines[len], pipe); 94 + if (ret) 95 + return ret; 96 + len++; 97 + 98 + return drm_plane_create_color_pipeline_property(plane, pipelines, len); 99 + }

+14

drivers/gpu/drm/i915/display/intel_color_pipeline.h

··· 1 + /* SPDX-License-Identifier: MIT */ 2 + /* 3 + * Copyright © 2025 Intel Corporation 4 + */ 5 + 6 + #ifndef __INTEL_COLOR_PIPELINE_H__ 7 + #define __INTEL_COLOR_PIPELINE_H__ 8 + 9 + struct drm_plane; 10 + enum pipe; 11 + 12 + int intel_color_pipeline_plane_init(struct drm_plane *plane, enum pipe pipe); 13 + 14 + #endif /* __INTEL_COLOR_PIPELINE_H__ */

+29

drivers/gpu/drm/i915/display/intel_color_regs.h

··· 316 316 #define SKL_BOTTOM_COLOR_CSC_ENABLE REG_BIT(30) 317 317 #define SKL_BOTTOM_COLOR(pipe) _MMIO_PIPE(pipe, _SKL_BOTTOM_COLOR_A, _SKL_BOTTOM_COLOR_B) 318 318 319 + /* 3D LUT */ 320 + #define _LUT_3D_CTL_A 0x490A4 321 + #define _LUT_3D_CTL_B 0x491A4 322 + #define LUT_3D_CTL(pipe) _MMIO_PIPE(pipe, _LUT_3D_CTL_A, _LUT_3D_CTL_B) 323 + #define LUT_3D_ENABLE REG_BIT(31) 324 + #define LUT_3D_READY REG_BIT(30) 325 + #define LUT_3D_BINDING_MASK REG_GENMASK(23, 22) 326 + #define LUT_3D_BIND_PIPE REG_FIELD_PREP(LUT_3D_BINDING_MASK, 0) 327 + #define LUT_3D_BIND_PLANE_1 REG_FIELD_PREP(LUT_3D_BINDING_MASK, 1) 328 + #define LUT_3D_BIND_PLANE_2 REG_FIELD_PREP(LUT_3D_BINDING_MASK, 2) 329 + #define LUT_3D_BIND_PLANE_3 REG_FIELD_PREP(LUT_3D_BINDING_MASK, 3) 330 + 331 + #define _LUT_3D_INDEX_A 0x490A8 332 + #define _LUT_3D_INDEX_B 0x491A8 333 + #define LUT_3D_INDEX(pipe) _MMIO_PIPE(pipe, _LUT_3D_INDEX_A, _LUT_3D_INDEX_B) 334 + #define LUT_3D_AUTO_INCREMENT REG_BIT(13) 335 + #define LUT_3D_INDEX_VALUE_MASK REG_GENMASK(12, 0) 336 + #define LUT_3D_INDEX_VALUE(x) REG_FIELD_PREP(LUT_3D_INDEX_VALUE_MASK, (x)) 337 + 338 + #define _LUT_3D_DATA_A 0x490AC 339 + #define _LUT_3D_DATA_B 0x491AC 340 + #define LUT_3D_DATA(pipe) _MMIO_PIPE(pipe, _LUT_3D_DATA_A, _LUT_3D_DATA_B) 341 + #define LUT_3D_DATA_RED_MASK REG_GENMASK(29, 20) 342 + #define LUT_3D_DATA_GREEN_MASK REG_GENMASK(19, 10) 343 + #define LUT_3D_DATA_BLUE_MASK REG_GENMASK(9, 0) 344 + #define LUT_3D_DATA_RED(x) REG_FIELD_PREP(LUT_3D_DATA_RED_MASK, (x)) 345 + #define LUT_3D_DATA_GREEN(x) REG_FIELD_PREP(LUT_3D_DATA_GREEN_MASK, (x)) 346 + #define LUT_3D_DATA_BLUE(x) REG_FIELD_PREP(LUT_3D_DATA_BLUE_MASK, (x)) 347 + 319 348 #endif /* __INTEL_COLOR_REGS_H__ */

+35

drivers/gpu/drm/i915/display/intel_colorop.c

··· 1 + // SPDX-License-Identifier: MIT 2 + /* 3 + * Copyright © 2025 Intel Corporation 4 + */ 5 + #include "intel_colorop.h" 6 + 7 + struct intel_colorop *to_intel_colorop(struct drm_colorop *colorop) 8 + { 9 + return container_of(colorop, struct intel_colorop, base); 10 + } 11 + 12 + struct intel_colorop *intel_colorop_alloc(void) 13 + { 14 + struct intel_colorop *colorop; 15 + 16 + colorop = kzalloc(sizeof(*colorop), GFP_KERNEL); 17 + if (!colorop) 18 + return ERR_PTR(-ENOMEM); 19 + 20 + return colorop; 21 + } 22 + 23 + struct intel_colorop *intel_colorop_create(enum intel_color_block id) 24 + { 25 + struct intel_colorop *colorop; 26 + 27 + colorop = intel_colorop_alloc(); 28 + 29 + if (IS_ERR(colorop)) 30 + return colorop; 31 + 32 + colorop->id = id; 33 + 34 + return colorop; 35 + }

+15

drivers/gpu/drm/i915/display/intel_colorop.h

··· 1 + /* SPDX-License-Identifier: MIT */ 2 + /* 3 + * Copyright © 2025 Intel Corporation 4 + */ 5 + 6 + #ifndef __INTEL_COLOROP_H__ 7 + #define __INTEL_COLOROP_H__ 8 + 9 + #include "intel_display_types.h" 10 + 11 + struct intel_colorop *to_intel_colorop(struct drm_colorop *colorop); 12 + struct intel_colorop *intel_colorop_alloc(void); 13 + struct intel_colorop *intel_colorop_create(enum intel_color_block id); 14 + 15 + #endif /* __INTEL_COLOROP_H__ */

+4 -1

drivers/gpu/drm/i915/display/intel_display.c

··· 7304 7304 struct intel_display *display = to_intel_display(state); 7305 7305 struct intel_crtc_state *new_crtc_state = 7306 7306 intel_atomic_get_new_crtc_state(state, crtc); 7307 + unsigned int size = new_crtc_state->plane_color_changed ? 8192 : 1024; 7307 7308 7308 7309 if (!new_crtc_state->use_flipq && 7309 7310 !new_crtc_state->use_dsb && ··· 7315 7314 * Rough estimate: 7316 7315 * ~64 registers per each plane * 8 planes = 512 7317 7316 * Double that for pipe stuff and other overhead. 7317 + * ~4913 registers for 3DLUT 7318 + * ~200 color registers * 3 HDR planes 7318 7319 */ 7319 7320 new_crtc_state->dsb_commit = intel_dsb_prepare(state, crtc, INTEL_DSB_0, 7320 7321 new_crtc_state->use_dsb || 7321 - new_crtc_state->use_flipq ? 1024 : 16); 7322 + new_crtc_state->use_flipq ? size : 16); 7322 7323 if (!new_crtc_state->dsb_commit) { 7323 7324 new_crtc_state->use_flipq = false; 7324 7325 new_crtc_state->use_dsb = false;

+9

drivers/gpu/drm/i915/display/intel_display_limits.h

··· 138 138 HPD_NUM_PINS 139 139 }; 140 140 141 + enum intel_color_block { 142 + INTEL_PLANE_CB_PRE_CSC_LUT, 143 + INTEL_PLANE_CB_CSC, 144 + INTEL_PLANE_CB_POST_CSC_LUT, 145 + INTEL_PLANE_CB_3DLUT, 146 + 147 + INTEL_CB_MAX 148 + }; 149 + 141 150 #endif /* __INTEL_DISPLAY_LIMITS_H__ */

+9

drivers/gpu/drm/i915/display/intel_display_types.h

··· 646 646 enum drm_color_encoding color_encoding; 647 647 enum drm_color_range color_range; 648 648 enum drm_scaling_filter scaling_filter; 649 + struct drm_property_blob *ctm, *degamma_lut, *gamma_lut, *lut_3d; 649 650 } hw; 650 651 651 652 struct i915_vma *ggtt_vma; ··· 1392 1391 u8 silence_period_sym_clocks; 1393 1392 u8 lfps_half_cycle_num_of_syms; 1394 1393 } alpm_state; 1394 + 1395 + /* to track changes in plane color blocks */ 1396 + bool plane_color_changed; 1395 1397 }; 1396 1398 1397 1399 enum intel_pipe_crc_source { ··· 1987 1983 enum pipe pipe; 1988 1984 struct intel_digital_port *primary; 1989 1985 struct intel_connector *connector; 1986 + }; 1987 + 1988 + struct intel_colorop { 1989 + struct drm_colorop base; 1990 + enum intel_color_block id; 1990 1991 }; 1991 1992 1992 1993 static inline struct intel_encoder *

+55

drivers/gpu/drm/i915/display/intel_plane.c

··· 49 49 #include "i9xx_plane_regs.h" 50 50 #include "intel_cdclk.h" 51 51 #include "intel_cursor.h" 52 + #include "intel_colorop.h" 52 53 #include "intel_display_rps.h" 53 54 #include "intel_display_trace.h" 54 55 #include "intel_display_types.h" ··· 337 336 *damage = drm_plane_state_src(&new_uapi_plane_state->uapi); 338 337 } 339 338 339 + static bool 340 + intel_plane_colorop_replace_blob(struct intel_plane_state *plane_state, 341 + struct intel_colorop *intel_colorop, 342 + struct drm_property_blob *blob) 343 + { 344 + if (intel_colorop->id == INTEL_PLANE_CB_CSC) 345 + return drm_property_replace_blob(&plane_state->hw.ctm, blob); 346 + else if (intel_colorop->id == INTEL_PLANE_CB_PRE_CSC_LUT) 347 + return drm_property_replace_blob(&plane_state->hw.degamma_lut, blob); 348 + else if (intel_colorop->id == INTEL_PLANE_CB_POST_CSC_LUT) 349 + return drm_property_replace_blob(&plane_state->hw.gamma_lut, blob); 350 + else if (intel_colorop->id == INTEL_PLANE_CB_3DLUT) 351 + return drm_property_replace_blob(&plane_state->hw.lut_3d, blob); 352 + 353 + return false; 354 + } 355 + 356 + static void 357 + intel_plane_color_copy_uapi_to_hw_state(struct intel_plane_state *plane_state, 358 + const struct intel_plane_state *from_plane_state, 359 + struct intel_crtc *crtc) 360 + { 361 + struct drm_colorop *iter_colorop, *colorop; 362 + struct drm_colorop_state *new_colorop_state; 363 + struct drm_atomic_state *state = plane_state->uapi.state; 364 + struct intel_colorop *intel_colorop; 365 + struct drm_property_blob *blob; 366 + struct intel_atomic_state *intel_atomic_state = to_intel_atomic_state(state); 367 + struct intel_crtc_state *new_crtc_state = intel_atomic_state ? 368 + intel_atomic_get_new_crtc_state(intel_atomic_state, crtc) : NULL; 369 + bool changed = false; 370 + int i = 0; 371 + 372 + iter_colorop = plane_state->uapi.color_pipeline; 373 + 374 + while (iter_colorop) { 375 + for_each_new_colorop_in_state(state, colorop, new_colorop_state, i) { 376 + if (new_colorop_state->colorop == iter_colorop) { 377 + blob = new_colorop_state->bypass ? NULL : new_colorop_state->data; 378 + intel_colorop = to_intel_colorop(colorop); 379 + changed |= intel_plane_colorop_replace_blob(plane_state, 380 + intel_colorop, 381 + blob); 382 + } 383 + } 384 + iter_colorop = iter_colorop->next; 385 + } 386 + 387 + if (new_crtc_state && changed) 388 + new_crtc_state->plane_color_changed = true; 389 + } 390 + 340 391 void intel_plane_copy_uapi_to_hw_state(struct intel_plane_state *plane_state, 341 392 const struct intel_plane_state *from_plane_state, 342 393 struct intel_crtc *crtc) ··· 417 364 418 365 plane_state->uapi.src = drm_plane_state_src(&from_plane_state->uapi); 419 366 plane_state->uapi.dst = drm_plane_state_dest(&from_plane_state->uapi); 367 + 368 + intel_plane_color_copy_uapi_to_hw_state(plane_state, from_plane_state, crtc); 420 369 } 421 370 422 371 void intel_plane_copy_hw_state(struct intel_plane_state *plane_state,

+21

drivers/gpu/drm/i915/display/skl_universal_plane.c

··· 11 11 12 12 #include "pxp/intel_pxp.h" 13 13 #include "intel_bo.h" 14 + #include "intel_color.h" 15 + #include "intel_color_pipeline.h" 14 16 #include "intel_de.h" 15 17 #include "intel_display_irq.h" 16 18 #include "intel_display_regs.h" ··· 1277 1275 if (plane_state->force_black) 1278 1276 plane_color_ctl |= PLANE_COLOR_PLANE_CSC_ENABLE; 1279 1277 1278 + if (plane_state->hw.degamma_lut) 1279 + plane_color_ctl |= PLANE_COLOR_PRE_CSC_GAMMA_ENABLE; 1280 + 1281 + if (plane_state->hw.ctm) 1282 + plane_color_ctl |= PLANE_COLOR_PLANE_CSC_ENABLE; 1283 + 1284 + if (plane_state->hw.gamma_lut) { 1285 + plane_color_ctl &= ~PLANE_COLOR_PLANE_GAMMA_DISABLE; 1286 + if (drm_color_lut32_size(plane_state->hw.gamma_lut) != 32) 1287 + plane_color_ctl |= PLANE_COLOR_POST_CSC_GAMMA_MULTSEG_ENABLE; 1288 + } 1289 + 1280 1290 return plane_color_ctl; 1281 1291 } 1282 1292 ··· 1570 1556 plane_color_ctl = plane_state->color_ctl | 1571 1557 glk_plane_color_ctl_crtc(crtc_state); 1572 1558 1559 + intel_color_plane_program_pipeline(dsb, plane_state); 1560 + 1573 1561 /* The scaler will handle the output position */ 1574 1562 if (plane_state->scaler_id >= 0) { 1575 1563 crtc_x = 0; ··· 1672 1656 skl_program_plane_scaler(dsb, plane, crtc_state, plane_state); 1673 1657 1674 1658 icl_plane_update_sel_fetch_arm(dsb, plane, crtc_state, plane_state); 1659 + 1660 + intel_color_plane_commit_arm(dsb, plane_state); 1675 1661 1676 1662 /* 1677 1663 * In order to have FBC for fp16 formats pixel normalizer block must be ··· 3018 3000 BIT(DRM_COLOR_YCBCR_FULL_RANGE), 3019 3001 DRM_COLOR_YCBCR_BT709, 3020 3002 DRM_COLOR_YCBCR_LIMITED_RANGE); 3003 + 3004 + if (DISPLAY_VER(display) >= 12) 3005 + intel_color_pipeline_plane_init(&plane->base, pipe); 3021 3006 3022 3007 drm_plane_create_alpha_property(&plane->base); 3023 3008 drm_plane_create_blend_mode_property(&plane->base,

+115

drivers/gpu/drm/i915/display/skl_universal_plane_regs.h

··· 254 254 #define PLANE_COLOR_PIPE_CSC_ENABLE REG_BIT(23) /* Pre-ICL */ 255 255 #define PLANE_COLOR_PLANE_CSC_ENABLE REG_BIT(21) /* ICL+ */ 256 256 #define PLANE_COLOR_INPUT_CSC_ENABLE REG_BIT(20) /* ICL+ */ 257 + #define PLANE_COLOR_POST_CSC_GAMMA_MULTSEG_ENABLE REG_BIT(15) /* TGL+ */ 258 + #define PLANE_COLOR_PRE_CSC_GAMMA_ENABLE REG_BIT(14) 257 259 #define PLANE_COLOR_CSC_MODE_MASK REG_GENMASK(19, 17) 258 260 #define PLANE_COLOR_CSC_MODE_BYPASS REG_FIELD_PREP(PLANE_COLOR_CSC_MODE_MASK, 0) 259 261 #define PLANE_COLOR_CSC_MODE_YUV601_TO_RGB601 REG_FIELD_PREP(PLANE_COLOR_CSC_MODE_MASK, 1) ··· 291 289 #define PLANE_INPUT_CSC_POSTOFF(pipe, plane, index) _MMIO_SKL_PLANE_DW((pipe), (plane), (index), \ 292 290 _PLANE_INPUT_CSC_POSTOFF_HI_1_A, _PLANE_INPUT_CSC_POSTOFF_HI_1_B, \ 293 291 _PLANE_INPUT_CSC_POSTOFF_HI_2_A, _PLANE_INPUT_CSC_POSTOFF_HI_2_B) 292 + 293 + #define _MMIO_PLANE_GAMC(plane, i, a, b) _MMIO(_PIPE(plane, a, b) + (i) * 4) 294 + 295 + #define _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_1_A 0x70160 296 + #define _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_1_B 0x71160 297 + #define _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_2_A 0x70260 298 + #define _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_2_B 0x71260 299 + #define _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_1(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_1_A, \ 300 + _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_1_B) 301 + #define _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_2(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_2_A, \ 302 + _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_2_B) 303 + #define PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH(pipe, plane, i) _MMIO_PLANE_GAMC(plane, i, _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_1(pipe), \ 304 + _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_2(pipe)) 305 + 306 + #define _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_1_A 0x70164 307 + #define _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_1_B 0x71164 308 + #define _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_2_A 0x70264 309 + #define _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_2_B 0x71264 310 + #define _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_1(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_1_A, \ 311 + _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_1_B) 312 + #define _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_2(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_2_A, \ 313 + _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_2_B) 314 + #define PLANE_POST_CSC_GAMC_SEG0_DATA_ENH(pipe, plane, i) _MMIO_PLANE_GAMC(plane, i, _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_1(pipe), \ 315 + _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_2(pipe)) 316 + 317 + #define _PLANE_POST_CSC_GAMC_INDEX_ENH_1_A 0x701d8 318 + #define _PLANE_POST_CSC_GAMC_INDEX_ENH_1_B 0x711d8 319 + #define _PLANE_POST_CSC_GAMC_INDEX_ENH_2_A 0x702d8 320 + #define _PLANE_POST_CSC_GAMC_INDEX_ENH_2_B 0x712d8 321 + #define _PLANE_POST_CSC_GAMC_INDEX_ENH_1(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_INDEX_ENH_1_A, \ 322 + _PLANE_POST_CSC_GAMC_INDEX_ENH_1_B) 323 + #define _PLANE_POST_CSC_GAMC_INDEX_ENH_2(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_INDEX_ENH_2_A, \ 324 + _PLANE_POST_CSC_GAMC_INDEX_ENH_2_B) 325 + #define PLANE_POST_CSC_GAMC_INDEX_ENH(pipe, plane, i) _MMIO_PLANE_GAMC(plane, i, _PLANE_POST_CSC_GAMC_INDEX_ENH_1(pipe), \ 326 + _PLANE_POST_CSC_GAMC_INDEX_ENH_2(pipe)) 327 + 328 + #define _PLANE_POST_CSC_GAMC_DATA_ENH_1_A 0x701dc 329 + #define _PLANE_POST_CSC_GAMC_DATA_ENH_1_B 0x711dc 330 + #define _PLANE_POST_CSC_GAMC_DATA_ENH_2_A 0x702dc 331 + #define _PLANE_POST_CSC_GAMC_DATA_ENH_2_B 0x712dc 332 + #define _PLANE_POST_CSC_GAMC_DATA_ENH_1(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_DATA_ENH_1_A, \ 333 + _PLANE_POST_CSC_GAMC_DATA_ENH_1_B) 334 + #define _PLANE_POST_CSC_GAMC_DATA_ENH_2(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_DATA_ENH_2_A, \ 335 + _PLANE_POST_CSC_GAMC_DATA_ENH_2_B) 336 + #define PLANE_POST_CSC_GAMC_DATA_ENH(pipe, plane, i) _MMIO_PLANE_GAMC(plane, i, _PLANE_POST_CSC_GAMC_DATA_ENH_1(pipe), \ 337 + _PLANE_POST_CSC_GAMC_DATA_ENH_2(pipe)) 338 + 339 + #define _PLANE_POST_CSC_GAMC_INDEX_1_A 0x704d8 340 + #define _PLANE_POST_CSC_GAMC_INDEX_1_B 0x714d8 341 + #define _PLANE_POST_CSC_GAMC_INDEX_2_A 0x705d8 342 + #define _PLANE_POST_CSC_GAMC_INDEX_2_B 0x715d8 343 + #define _PLANE_POST_CSC_GAMC_INDEX_1(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_INDEX_1_A, \ 344 + _PLANE_POST_CSC_GAMC_INDEX_1_B) 345 + #define _PLANE_POST_CSC_GAMC_INDEX_2(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_INDEX_2_A, \ 346 + _PLANE_POST_CSC_GAMC_INDEX_2_B) 347 + #define PLANE_POST_CSC_GAMC_INDEX(pipe, plane, i) _MMIO_PLANE_GAMC(plane, i, _PLANE_POST_CSC_GAMC_INDEX_1(pipe), \ 348 + _PLANE_POST_CSC_GAMC_INDEX_2(pipe)) 349 + 350 + #define _PLANE_POST_CSC_GAMC_DATA_1_A 0x704dc 351 + #define _PLANE_POST_CSC_GAMC_DATA_1_B 0x714dc 352 + #define _PLANE_POST_CSC_GAMC_DATA_2_A 0x705dc 353 + #define _PLANE_POST_CSC_GAMC_DATA_2_B 0x715dc 354 + #define _PLANE_POST_CSC_GAMC_DATA_1(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_DATA_1_A, \ 355 + _PLANE_POST_CSC_GAMC_DATA_1_B) 356 + #define _PLANE_POST_CSC_GAMC_DATA_2(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_DATA_2_A, \ 357 + _PLANE_POST_CSC_GAMC_DATA_2_B) 358 + #define PLANE_POST_CSC_GAMC_DATA(pipe, plane, i) _MMIO_PLANE_GAMC(plane, i, _PLANE_POST_CSC_GAMC_DATA_1(pipe), \ 359 + _PLANE_POST_CSC_GAMC_DATA_2(pipe)) 360 + 361 + #define _PLANE_PRE_CSC_GAMC_INDEX_ENH_1_A 0x701d0 362 + #define _PLANE_PRE_CSC_GAMC_INDEX_ENH_1_B 0x711d0 363 + #define _PLANE_PRE_CSC_GAMC_INDEX_ENH_2_A 0x702d0 364 + #define _PLANE_PRE_CSC_GAMC_INDEX_ENH_2_B 0x712d0 365 + #define _PLANE_PRE_CSC_GAMC_INDEX_ENH_1(pipe) _PIPE(pipe, _PLANE_PRE_CSC_GAMC_INDEX_ENH_1_A, \ 366 + _PLANE_PRE_CSC_GAMC_INDEX_ENH_1_B) 367 + #define _PLANE_PRE_CSC_GAMC_INDEX_ENH_2(pipe) _PIPE(pipe, _PLANE_PRE_CSC_GAMC_INDEX_ENH_2_A, \ 368 + _PLANE_PRE_CSC_GAMC_INDEX_ENH_2_B) 369 + #define PLANE_PRE_CSC_GAMC_INDEX_ENH(pipe, plane, i) _MMIO_PLANE_GAMC(plane, i, _PLANE_PRE_CSC_GAMC_INDEX_ENH_1(pipe), \ 370 + _PLANE_PRE_CSC_GAMC_INDEX_ENH_2(pipe)) 371 + #define PLANE_PAL_PREC_AUTO_INCREMENT REG_BIT(10) 372 + 373 + #define _PLANE_PRE_CSC_GAMC_DATA_ENH_1_A 0x701d4 374 + #define _PLANE_PRE_CSC_GAMC_DATA_ENH_1_B 0x711d4 375 + #define _PLANE_PRE_CSC_GAMC_DATA_ENH_2_A 0x702d4 376 + #define _PLANE_PRE_CSC_GAMC_DATA_ENH_2_B 0x712d4 377 + #define _PLANE_PRE_CSC_GAMC_DATA_ENH_1(pipe) _PIPE(pipe, _PLANE_PRE_CSC_GAMC_DATA_ENH_1_A, \ 378 + _PLANE_PRE_CSC_GAMC_DATA_ENH_1_B) 379 + #define _PLANE_PRE_CSC_GAMC_DATA_ENH_2(pipe) _PIPE(pipe, _PLANE_PRE_CSC_GAMC_DATA_ENH_2_A, \ 380 + _PLANE_PRE_CSC_GAMC_DATA_ENH_2_B) 381 + #define PLANE_PRE_CSC_GAMC_DATA_ENH(pipe, plane, i) _MMIO_PLANE_GAMC(plane, i, _PLANE_PRE_CSC_GAMC_DATA_ENH_1(pipe), \ 382 + _PLANE_PRE_CSC_GAMC_DATA_ENH_2(pipe)) 383 + 384 + #define _PLANE_PRE_CSC_GAMC_INDEX_1_A 0x704d0 385 + #define _PLANE_PRE_CSC_GAMC_INDEX_1_B 0x714d0 386 + #define _PLANE_PRE_CSC_GAMC_INDEX_2_A 0x705d0 387 + #define _PLANE_PRE_CSC_GAMC_INDEX_2_B 0x715d0 388 + #define _PLANE_PRE_CSC_GAMC_INDEX_1(pipe) _PIPE(pipe, _PLANE_PRE_CSC_GAMC_INDEX_1_A, \ 389 + _PLANE_PRE_CSC_GAMC_INDEX_1_B) 390 + #define _PLANE_PRE_CSC_GAMC_INDEX_2(pipe) _PIPE(pipe, _PLANE_PRE_CSC_GAMC_INDEX_2_A, \ 391 + _PLANE_PRE_CSC_GAMC_INDEX_2_B) 392 + #define PLANE_PRE_CSC_GAMC_INDEX(pipe, plane, i) _MMIO_PLANE_GAMC(plane, i, _PLANE_PRE_CSC_GAMC_INDEX_1(pipe), \ 393 + _PLANE_PRE_CSC_GAMC_INDEX_2(pipe)) 394 + 395 + #define _PLANE_PRE_CSC_GAMC_DATA_1_A 0x704d4 396 + #define _PLANE_PRE_CSC_GAMC_DATA_1_B 0x714d4 397 + #define _PLANE_PRE_CSC_GAMC_DATA_2_A 0x705d4 398 + #define _PLANE_PRE_CSC_GAMC_DATA_2_B 0x715d4 399 + #define _PLANE_PRE_CSC_GAMC_DATA_1(pipe) _PIPE(pipe, _PLANE_PRE_CSC_GAMC_DATA_1_A, \ 400 + _PLANE_PRE_CSC_GAMC_DATA_1_B) 401 + #define _PLANE_PRE_CSC_GAMC_DATA_2(pipe) _PIPE(pipe, _PLANE_PRE_CSC_GAMC_DATA_2_A, \ 402 + _PLANE_PRE_CSC_GAMC_DATA_2_B) 403 + #define PLANE_PRE_CSC_GAMC_DATA(pipe, plane, i) _MMIO_PLANE_GAMC(plane, i, _PLANE_PRE_CSC_GAMC_DATA_1(pipe), \ 404 + _PLANE_PRE_CSC_GAMC_DATA_2(pipe)) 294 405 295 406 #define _PLANE_CSC_RY_GY_1_A 0x70210 296 407 #define _PLANE_CSC_RY_GY_2_A 0x70310

+6

drivers/gpu/drm/xe/Makefile

··· 184 184 xe_sriov_pf_sysfs.o \ 185 185 xe_tile_sriov_pf_debugfs.o 186 186 187 + ifdef CONFIG_XE_VFIO_PCI 188 + xe-$(CONFIG_PCI_IOV) += xe_sriov_vfio.o 189 + endif 190 + 187 191 # include helpers for tests even when XE is built-in 188 192 ifdef CONFIG_DRM_XE_KUNIT_TEST 189 193 xe-y += tests/xe_kunit_helpers.o ··· 246 242 i915-display/intel_cdclk.o \ 247 243 i915-display/intel_cmtg.o \ 248 244 i915-display/intel_color.o \ 245 + i915-display/intel_colorop.o \ 246 + i915-display/intel_color_pipeline.o \ 249 247 i915-display/intel_combo_phy.o \ 250 248 i915-display/intel_connector.o \ 251 249 i915-display/intel_crtc.o \

+3 -2

drivers/gpu/drm/xe/xe_gpu_scheduler.h

··· 54 54 static inline void xe_sched_resubmit_jobs(struct xe_gpu_scheduler *sched) 55 55 { 56 56 struct drm_sched_job *s_job; 57 + bool restore_replay = false; 57 58 58 59 list_for_each_entry(s_job, &sched->base.pending_list, list) { 59 60 struct drm_sched_fence *s_fence = s_job->s_fence; 60 61 struct dma_fence *hw_fence = s_fence->parent; 61 62 62 - if (to_xe_sched_job(s_job)->skip_emit || 63 - (hw_fence && !dma_fence_is_signaled(hw_fence))) 63 + restore_replay |= to_xe_sched_job(s_job)->restore_replay; 64 + if (restore_replay || (hw_fence && !dma_fence_is_signaled(hw_fence))) 64 65 sched->base.ops->run_job(s_job); 65 66 } 66 67 }

+1 -1

drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c

··· 711 711 if (num_vfs > 56) 712 712 return SZ_64M - SZ_8M; 713 713 714 - return rounddown_pow_of_two(shareable / num_vfs); 714 + return rounddown_pow_of_two(div_u64(shareable, num_vfs)); 715 715 } 716 716 717 717 /**

+9

drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c

··· 17 17 #include "xe_gt_sriov_pf_helpers.h" 18 18 #include "xe_gt_sriov_pf_migration.h" 19 19 #include "xe_gt_sriov_printk.h" 20 + #include "xe_guc.h" 20 21 #include "xe_guc_buf.h" 21 22 #include "xe_guc_ct.h" 22 23 #include "xe_migrate.h" ··· 1024 1023 ptr_ring_cleanup(r, destroy_pf_packet); 1025 1024 } 1026 1025 1026 + static void pf_gt_migration_check_support(struct xe_gt *gt) 1027 + { 1028 + if (GUC_FIRMWARE_VER(&gt->uc.guc) < MAKE_GUC_VER(70, 54, 0)) 1029 + xe_sriov_pf_migration_disable(gt_to_xe(gt), "requires GuC version >= 70.54.0"); 1030 + } 1031 + 1027 1032 /** 1028 1033 * xe_gt_sriov_pf_migration_init() - Initialize support for VF migration. 1029 1034 * @gt: the &xe_gt ··· 1045 1038 int err; 1046 1039 1047 1040 xe_gt_assert(gt, IS_SRIOV_PF(xe)); 1041 + 1042 + pf_gt_migration_check_support(gt); 1048 1043 1049 1044 if (!pf_migration_supported(gt)) 1050 1045 return 0;

+30 -17

drivers/gpu/drm/xe/xe_guc_submit.c

··· 822 822 823 823 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); 824 824 825 - if (!job->skip_emit || job->last_replay) { 825 + if (!job->restore_replay || job->last_replay) { 826 826 if (xe_exec_queue_is_parallel(q)) 827 827 wq_item_append(q); 828 828 else ··· 881 881 if (!killed_or_banned_or_wedged && !xe_sched_job_is_error(job)) { 882 882 if (!exec_queue_registered(q)) 883 883 register_exec_queue(q, GUC_CONTEXT_NORMAL); 884 - if (!job->skip_emit) 884 + if (!job->restore_replay) 885 885 q->ring_ops->emit_job(job); 886 886 submit_exec_queue(q, job); 887 - job->skip_emit = false; 887 + job->restore_replay = false; 888 888 } 889 889 890 890 /* ··· 2112 2112 q->guc->resume_time = 0; 2113 2113 } 2114 2114 2115 + static void lrc_parallel_clear(struct xe_lrc *lrc) 2116 + { 2117 + struct xe_device *xe = gt_to_xe(lrc->gt); 2118 + struct iosys_map map = xe_lrc_parallel_map(lrc); 2119 + int i; 2120 + 2121 + for (i = 0; i < WQ_SIZE / sizeof(u32); ++i) 2122 + parallel_write(xe, map, wq[i], 2123 + FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | 2124 + FIELD_PREP(WQ_LEN_MASK, 0)); 2125 + } 2126 + 2115 2127 /* 2116 2128 * This function is quite complex but only real way to ensure no state is lost 2117 2129 * during VF resume flows. The function scans the queue state, make adjustments ··· 2147 2135 guc_exec_queue_revert_pending_state_change(guc, q); 2148 2136 2149 2137 if (xe_exec_queue_is_parallel(q)) { 2150 - struct xe_device *xe = guc_to_xe(guc); 2151 - struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); 2138 + /* Pairs with WRITE_ONCE in __xe_exec_queue_init */ 2139 + struct xe_lrc *lrc = READ_ONCE(q->lrc[0]); 2152 2140 2153 2141 /* 2154 2142 * NOP existing WQ commands that may contain stale GGTT ··· 2156 2144 * seems to get confused if the WQ head/tail pointers are 2157 2145 * adjusted. 2158 2146 */ 2159 - for (i = 0; i < WQ_SIZE / sizeof(u32); ++i) 2160 - parallel_write(xe, map, wq[i], 2161 - FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | 2162 - FIELD_PREP(WQ_LEN_MASK, 0)); 2147 + if (lrc) 2148 + lrc_parallel_clear(lrc); 2163 2149 } 2164 2150 2165 2151 job = xe_sched_first_pending_job(sched); 2166 2152 if (job) { 2153 + job->restore_replay = true; 2154 + 2167 2155 /* 2168 2156 * Adjust software tail so jobs submitted overwrite previous 2169 2157 * position in ring buffer with new GGTT addresses. ··· 2253 2241 struct xe_exec_queue *q) 2254 2242 { 2255 2243 struct xe_gpu_scheduler *sched = &q->guc->sched; 2256 - struct drm_sched_job *s_job; 2257 2244 struct xe_sched_job *job = NULL; 2245 + bool restore_replay = false; 2258 2246 2259 - list_for_each_entry(s_job, &sched->base.pending_list, list) { 2260 - job = to_xe_sched_job(s_job); 2247 + list_for_each_entry(job, &sched->base.pending_list, drm.list) { 2248 + restore_replay |= job->restore_replay; 2249 + if (restore_replay) { 2250 + xe_gt_dbg(guc_to_gt(guc), "Replay JOB - guc_id=%d, seqno=%d", 2251 + q->guc->id, xe_sched_job_seqno(job)); 2261 2252 2262 - xe_gt_dbg(guc_to_gt(guc), "Replay JOB - guc_id=%d, seqno=%d", 2263 - q->guc->id, xe_sched_job_seqno(job)); 2264 - 2265 - q->ring_ops->emit_job(job); 2266 - job->skip_emit = true; 2253 + q->ring_ops->emit_job(job); 2254 + job->restore_replay = true; 2255 + } 2267 2256 } 2268 2257 2269 2258 if (job)

-1

drivers/gpu/drm/xe/xe_pagefault.c

··· 102 102 103 103 /* Lock VM and BOs dma-resv */ 104 104 xe_validation_ctx_init(&ctx, &vm->xe->val, &exec, (struct xe_val_flags) {}); 105 - drm_exec_init(&exec, 0, 0); 106 105 drm_exec_until_all_locked(&exec) { 107 106 err = xe_pagefault_begin(&exec, vma, tile->mem.vram, 108 107 needs_vram == 1);

+17

drivers/gpu/drm/xe/xe_pci.c

··· 1223 1223 #endif 1224 1224 }; 1225 1225 1226 + /** 1227 + * xe_pci_to_pf_device() - Get PF &xe_device. 1228 + * @pdev: the VF &pci_dev device 1229 + * 1230 + * Return: pointer to PF &xe_device, NULL otherwise. 1231 + */ 1232 + struct xe_device *xe_pci_to_pf_device(struct pci_dev *pdev) 1233 + { 1234 + struct drm_device *drm; 1235 + 1236 + drm = pci_iov_get_pf_drvdata(pdev, &xe_pci_driver); 1237 + if (IS_ERR(drm)) 1238 + return NULL; 1239 + 1240 + return to_xe_device(drm); 1241 + } 1242 + 1226 1243 int xe_register_pci_driver(void) 1227 1244 { 1228 1245 return pci_register_driver(&xe_pci_driver);

+3

drivers/gpu/drm/xe/xe_pci.h

··· 6 6 #ifndef _XE_PCI_H_ 7 7 #define _XE_PCI_H_ 8 8 9 + struct pci_dev; 10 + 9 11 int xe_register_pci_driver(void); 10 12 void xe_unregister_pci_driver(void); 13 + struct xe_device *xe_pci_to_pf_device(struct pci_dev *pdev); 11 14 12 15 #endif

+21

drivers/gpu/drm/xe/xe_pm.c

··· 726 726 /** 727 727 * xe_pm_runtime_get - Get a runtime_pm reference and resume synchronously 728 728 * @xe: xe device instance 729 + * 730 + * When possible, scope-based runtime PM (through guard(xe_pm_runtime)) is 731 + * be preferred over direct usage of this function. Manual get/put handling 732 + * should only be used when the function contains goto-based logic which 733 + * can break scope-based handling, or when the lifetime of the runtime PM 734 + * reference does not match a specific scope (e.g., runtime PM obtained in one 735 + * function and released in a different one). 729 736 */ 730 737 void xe_pm_runtime_get(struct xe_device *xe) 731 738 { ··· 764 757 /** 765 758 * xe_pm_runtime_get_ioctl - Get a runtime_pm reference before ioctl 766 759 * @xe: xe device instance 760 + * 761 + * When possible, scope-based runtime PM (through 762 + * ACQUIRE(xe_pm_runtime_ioctl, ...)) is be preferred over direct usage of this 763 + * function. Manual get/put handling should only be used when the function 764 + * contains goto-based logic which can break scope-based handling, or when the 765 + * lifetime of the runtime PM reference does not match a specific scope (e.g., 766 + * runtime PM obtained in one function and released in a different one). 767 767 * 768 768 * Returns: Any number greater than or equal to 0 for success, negative error 769 769 * code otherwise. ··· 841 827 * It will warn if not protected. 842 828 * The reference should be put back after this function regardless, since it 843 829 * will always bump the usage counter, regardless. 830 + * 831 + * When possible, scope-based runtime PM (through guard(xe_pm_runtime_noresume)) 832 + * is be preferred over direct usage of this function. Manual get/put handling 833 + * should only be used when the function contains goto-based logic which can 834 + * break scope-based handling, or when the lifetime of the runtime PM reference 835 + * does not match a specific scope (e.g., runtime PM obtained in one function 836 + * and released in a different one). 844 837 */ 845 838 void xe_pm_runtime_get_noresume(struct xe_device *xe) 846 839 {

+17

drivers/gpu/drm/xe/xe_pm.h

··· 6 6 #ifndef _XE_PM_H_ 7 7 #define _XE_PM_H_ 8 8 9 + #include <linux/cleanup.h> 9 10 #include <linux/pm_runtime.h> 10 11 11 12 #define DEFAULT_VRAM_THRESHOLD 300 /* in MB */ ··· 37 36 int xe_pm_block_on_suspend(struct xe_device *xe); 38 37 void xe_pm_might_block_on_suspend(void); 39 38 int xe_pm_module_init(void); 39 + 40 + static inline void __xe_pm_runtime_noop(struct xe_device *xe) {} 41 + 42 + DEFINE_GUARD(xe_pm_runtime, struct xe_device *, 43 + xe_pm_runtime_get(_T), xe_pm_runtime_put(_T)) 44 + DEFINE_GUARD(xe_pm_runtime_noresume, struct xe_device *, 45 + xe_pm_runtime_get_noresume(_T), xe_pm_runtime_put(_T)) 46 + DEFINE_GUARD_COND(xe_pm_runtime, _ioctl, xe_pm_runtime_get_ioctl(_T), _RET >= 0) 47 + 48 + /* 49 + * Used when a function needs to release runtime PM in all possible cases 50 + * and error paths, but the wakeref was already acquired by a different 51 + * function (i.e., get() has already happened so only a put() is needed). 52 + */ 53 + DEFINE_GUARD(xe_pm_runtime_release_only, struct xe_device *, 54 + __xe_pm_runtime_noop(_T), xe_pm_runtime_put(_T)); 40 55 41 56 #endif

+2 -2

drivers/gpu/drm/xe/xe_sched_job_types.h

··· 63 63 bool ring_ops_flush_tlb; 64 64 /** @ggtt: mapped in ggtt. */ 65 65 bool ggtt; 66 - /** @skip_emit: skip emitting the job */ 67 - bool skip_emit; 66 + /** @restore_replay: job being replayed for restore */ 67 + bool restore_replay; 68 68 /** @last_replay: last job being replayed */ 69 69 bool last_replay; 70 70 /** @ptrs: per instance pointers. */

+30 -5

drivers/gpu/drm/xe/xe_sriov_pf_migration.c

··· 46 46 { 47 47 xe_assert(xe, IS_SRIOV_PF(xe)); 48 48 49 - return xe->sriov.pf.migration.supported; 49 + return IS_ENABLED(CONFIG_DRM_XE_DEBUG) || !xe->sriov.pf.migration.disabled; 50 50 } 51 51 52 - static bool pf_check_migration_support(struct xe_device *xe) 52 + /** 53 + * xe_sriov_pf_migration_disable() - Turn off SR-IOV VF migration support on PF. 54 + * @xe: the &xe_device instance. 55 + * @fmt: format string for the log message, to be combined with following VAs. 56 + */ 57 + void xe_sriov_pf_migration_disable(struct xe_device *xe, const char *fmt, ...) 53 58 { 54 - /* XXX: for now this is for feature enabling only */ 55 - return IS_ENABLED(CONFIG_DRM_XE_DEBUG); 59 + struct va_format vaf; 60 + va_list va_args; 61 + 62 + xe_assert(xe, IS_SRIOV_PF(xe)); 63 + 64 + va_start(va_args, fmt); 65 + vaf.fmt = fmt; 66 + vaf.va = &va_args; 67 + xe_sriov_notice(xe, "migration %s: %pV\n", 68 + IS_ENABLED(CONFIG_DRM_XE_DEBUG) ? 69 + "missing prerequisite" : "disabled", 70 + &vaf); 71 + va_end(va_args); 72 + 73 + xe->sriov.pf.migration.disabled = true; 74 + } 75 + 76 + static void pf_migration_check_support(struct xe_device *xe) 77 + { 78 + if (!xe_device_has_memirq(xe)) 79 + xe_sriov_pf_migration_disable(xe, "requires memory-based IRQ support"); 56 80 } 57 81 58 82 static void pf_migration_cleanup(void *arg) ··· 101 77 102 78 xe_assert(xe, IS_SRIOV_PF(xe)); 103 79 104 - xe->sriov.pf.migration.supported = pf_check_migration_support(xe); 80 + pf_migration_check_support(xe); 81 + 105 82 if (!xe_sriov_pf_migration_supported(xe)) 106 83 return 0; 107 84

+1

drivers/gpu/drm/xe/xe_sriov_pf_migration.h

··· 14 14 15 15 int xe_sriov_pf_migration_init(struct xe_device *xe); 16 16 bool xe_sriov_pf_migration_supported(struct xe_device *xe); 17 + void xe_sriov_pf_migration_disable(struct xe_device *xe, const char *fmt, ...); 17 18 int xe_sriov_pf_migration_restore_produce(struct xe_device *xe, unsigned int vfid, 18 19 struct xe_sriov_packet *data); 19 20 struct xe_sriov_packet *

+2 -2

drivers/gpu/drm/xe/xe_sriov_pf_migration_types.h

··· 14 14 * struct xe_sriov_pf_migration - Xe device level VF migration data 15 15 */ 16 16 struct xe_sriov_pf_migration { 17 - /** @supported: indicates whether VF migration feature is supported */ 18 - bool supported; 17 + /** @disabled: indicates whether VF migration feature is disabled */ 18 + bool disabled; 19 19 }; 20 20 21 21 /**

+80

drivers/gpu/drm/xe/xe_sriov_vfio.c

··· 1 + // SPDX-License-Identifier: MIT 2 + /* 3 + * Copyright © 2025 Intel Corporation 4 + */ 5 + 6 + #include <drm/intel/xe_sriov_vfio.h> 7 + #include <linux/cleanup.h> 8 + 9 + #include "xe_pci.h" 10 + #include "xe_pm.h" 11 + #include "xe_sriov_pf_control.h" 12 + #include "xe_sriov_pf_helpers.h" 13 + #include "xe_sriov_pf_migration.h" 14 + 15 + struct xe_device *xe_sriov_vfio_get_pf(struct pci_dev *pdev) 16 + { 17 + return xe_pci_to_pf_device(pdev); 18 + } 19 + EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_get_pf, "xe-vfio-pci"); 20 + 21 + bool xe_sriov_vfio_migration_supported(struct xe_device *xe) 22 + { 23 + if (!IS_SRIOV_PF(xe)) 24 + return -EPERM; 25 + 26 + return xe_sriov_pf_migration_supported(xe); 27 + } 28 + EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_migration_supported, "xe-vfio-pci"); 29 + 30 + #define DEFINE_XE_SRIOV_VFIO_FUNCTION(_type, _func, _impl) \ 31 + _type xe_sriov_vfio_##_func(struct xe_device *xe, unsigned int vfid) \ 32 + { \ 33 + if (!IS_SRIOV_PF(xe)) \ 34 + return -EPERM; \ 35 + if (vfid == PFID || vfid > xe_sriov_pf_num_vfs(xe)) \ 36 + return -EINVAL; \ 37 + \ 38 + guard(xe_pm_runtime_noresume)(xe); \ 39 + \ 40 + return xe_sriov_pf_##_impl(xe, vfid); \ 41 + } \ 42 + EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_##_func, "xe-vfio-pci") 43 + 44 + DEFINE_XE_SRIOV_VFIO_FUNCTION(int, wait_flr_done, control_wait_flr); 45 + DEFINE_XE_SRIOV_VFIO_FUNCTION(int, suspend_device, control_pause_vf); 46 + DEFINE_XE_SRIOV_VFIO_FUNCTION(int, resume_device, control_resume_vf); 47 + DEFINE_XE_SRIOV_VFIO_FUNCTION(int, stop_copy_enter, control_trigger_save_vf); 48 + DEFINE_XE_SRIOV_VFIO_FUNCTION(int, stop_copy_exit, control_finish_save_vf); 49 + DEFINE_XE_SRIOV_VFIO_FUNCTION(int, resume_data_enter, control_trigger_restore_vf); 50 + DEFINE_XE_SRIOV_VFIO_FUNCTION(int, resume_data_exit, control_finish_restore_vf); 51 + DEFINE_XE_SRIOV_VFIO_FUNCTION(int, error, control_stop_vf); 52 + DEFINE_XE_SRIOV_VFIO_FUNCTION(ssize_t, stop_copy_size, migration_size); 53 + 54 + ssize_t xe_sriov_vfio_data_read(struct xe_device *xe, unsigned int vfid, 55 + char __user *buf, size_t len) 56 + { 57 + if (!IS_SRIOV_PF(xe)) 58 + return -EPERM; 59 + if (vfid == PFID || vfid > xe_sriov_pf_num_vfs(xe)) 60 + return -EINVAL; 61 + 62 + guard(xe_pm_runtime_noresume)(xe); 63 + 64 + return xe_sriov_pf_migration_read(xe, vfid, buf, len); 65 + } 66 + EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_data_read, "xe-vfio-pci"); 67 + 68 + ssize_t xe_sriov_vfio_data_write(struct xe_device *xe, unsigned int vfid, 69 + const char __user *buf, size_t len) 70 + { 71 + if (!IS_SRIOV_PF(xe)) 72 + return -EPERM; 73 + if (vfid == PFID || vfid > xe_sriov_pf_num_vfs(xe)) 74 + return -EINVAL; 75 + 76 + guard(xe_pm_runtime_noresume)(xe); 77 + 78 + return xe_sriov_pf_migration_write(xe, vfid, buf, len); 79 + } 80 + EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_data_write, "xe-vfio-pci");

+2

drivers/vfio/pci/Kconfig

··· 70 70 71 71 source "drivers/vfio/pci/qat/Kconfig" 72 72 73 + source "drivers/vfio/pci/xe/Kconfig" 74 + 73 75 endmenu

+2

drivers/vfio/pci/Makefile

··· 20 20 obj-$(CONFIG_NVGRACE_GPU_VFIO_PCI) += nvgrace-gpu/ 21 21 22 22 obj-$(CONFIG_QAT_VFIO_PCI) += qat/ 23 + 24 + obj-$(CONFIG_XE_VFIO_PCI) += xe/

+12

drivers/vfio/pci/xe/Kconfig

··· 1 + # SPDX-License-Identifier: GPL-2.0-only 2 + config XE_VFIO_PCI 3 + tristate "VFIO support for Intel Graphics" 4 + depends on DRM_XE && PCI_IOV 5 + select VFIO_PCI_CORE 6 + help 7 + This option enables device specific VFIO driver variant for Intel Graphics. 8 + In addition to generic VFIO PCI functionality, it implements VFIO 9 + migration uAPI allowing userspace to enable migration for 10 + Intel Graphics SR-IOV Virtual Functions supported by the Xe driver. 11 + 12 + If you don't know what to do here, say N.

+3

drivers/vfio/pci/xe/Makefile

··· 1 + # SPDX-License-Identifier: GPL-2.0-only 2 + obj-$(CONFIG_XE_VFIO_PCI) += xe-vfio-pci.o 3 + xe-vfio-pci-y := main.o

+573

drivers/vfio/pci/xe/main.c

··· 1 + // SPDX-License-Identifier: GPL-2.0-only 2 + /* 3 + * Copyright © 2025 Intel Corporation 4 + */ 5 + 6 + #include <linux/anon_inodes.h> 7 + #include <linux/delay.h> 8 + #include <linux/file.h> 9 + #include <linux/module.h> 10 + #include <linux/pci.h> 11 + #include <linux/sizes.h> 12 + #include <linux/types.h> 13 + #include <linux/vfio.h> 14 + #include <linux/vfio_pci_core.h> 15 + 16 + #include <drm/intel/xe_sriov_vfio.h> 17 + #include <drm/intel/pciids.h> 18 + 19 + struct xe_vfio_pci_migration_file { 20 + struct file *filp; 21 + /* serializes accesses to migration data */ 22 + struct mutex lock; 23 + struct xe_vfio_pci_core_device *xe_vdev; 24 + u8 disabled:1; 25 + }; 26 + 27 + struct xe_vfio_pci_core_device { 28 + struct vfio_pci_core_device core_device; 29 + struct xe_device *xe; 30 + /* PF internal control uses vfid index starting from 1 */ 31 + unsigned int vfid; 32 + u8 deferred_reset:1; 33 + /* protects migration state */ 34 + struct mutex state_mutex; 35 + enum vfio_device_mig_state mig_state; 36 + /* protects the reset_done flow */ 37 + spinlock_t reset_lock; 38 + struct xe_vfio_pci_migration_file *migf; 39 + }; 40 + 41 + #define xe_vdev_to_dev(xe_vdev) (&(xe_vdev)->core_device.pdev->dev) 42 + 43 + static void xe_vfio_pci_disable_file(struct xe_vfio_pci_migration_file *migf) 44 + { 45 + mutex_lock(&migf->lock); 46 + migf->disabled = true; 47 + mutex_unlock(&migf->lock); 48 + } 49 + 50 + static void xe_vfio_pci_put_file(struct xe_vfio_pci_core_device *xe_vdev) 51 + { 52 + xe_vfio_pci_disable_file(xe_vdev->migf); 53 + fput(xe_vdev->migf->filp); 54 + xe_vdev->migf = NULL; 55 + } 56 + 57 + static void xe_vfio_pci_reset(struct xe_vfio_pci_core_device *xe_vdev) 58 + { 59 + if (xe_vdev->migf) 60 + xe_vfio_pci_put_file(xe_vdev); 61 + 62 + xe_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING; 63 + } 64 + 65 + static void xe_vfio_pci_state_mutex_lock(struct xe_vfio_pci_core_device *xe_vdev) 66 + { 67 + mutex_lock(&xe_vdev->state_mutex); 68 + } 69 + 70 + /* 71 + * This function is called in all state_mutex unlock cases to 72 + * handle a 'deferred_reset' if exists. 73 + */ 74 + static void xe_vfio_pci_state_mutex_unlock(struct xe_vfio_pci_core_device *xe_vdev) 75 + { 76 + again: 77 + spin_lock(&xe_vdev->reset_lock); 78 + if (xe_vdev->deferred_reset) { 79 + xe_vdev->deferred_reset = false; 80 + spin_unlock(&xe_vdev->reset_lock); 81 + xe_vfio_pci_reset(xe_vdev); 82 + goto again; 83 + } 84 + mutex_unlock(&xe_vdev->state_mutex); 85 + spin_unlock(&xe_vdev->reset_lock); 86 + } 87 + 88 + static void xe_vfio_pci_reset_done(struct pci_dev *pdev) 89 + { 90 + struct xe_vfio_pci_core_device *xe_vdev = pci_get_drvdata(pdev); 91 + int ret; 92 + 93 + if (!pdev->is_virtfn) 94 + return; 95 + 96 + /* 97 + * VF FLR requires additional processing done by PF driver. 98 + * The processing is done after FLR is already finished from PCIe 99 + * perspective. 100 + * In order to avoid a scenario where VF is used while PF processing 101 + * is still in progress, additional synchronization point is needed. 102 + */ 103 + ret = xe_sriov_vfio_wait_flr_done(xe_vdev->xe, xe_vdev->vfid); 104 + if (ret) 105 + dev_err(&pdev->dev, "Failed to wait for FLR: %d\n", ret); 106 + 107 + if (!xe_vdev->vfid) 108 + return; 109 + 110 + /* 111 + * As the higher VFIO layers are holding locks across reset and using 112 + * those same locks with the mm_lock we need to prevent ABBA deadlock 113 + * with the state_mutex and mm_lock. 114 + * In case the state_mutex was taken already we defer the cleanup work 115 + * to the unlock flow of the other running context. 116 + */ 117 + spin_lock(&xe_vdev->reset_lock); 118 + xe_vdev->deferred_reset = true; 119 + if (!mutex_trylock(&xe_vdev->state_mutex)) { 120 + spin_unlock(&xe_vdev->reset_lock); 121 + return; 122 + } 123 + spin_unlock(&xe_vdev->reset_lock); 124 + xe_vfio_pci_state_mutex_unlock(xe_vdev); 125 + 126 + xe_vfio_pci_reset(xe_vdev); 127 + } 128 + 129 + static const struct pci_error_handlers xe_vfio_pci_err_handlers = { 130 + .reset_done = xe_vfio_pci_reset_done, 131 + .error_detected = vfio_pci_core_aer_err_detected, 132 + }; 133 + 134 + static int xe_vfio_pci_open_device(struct vfio_device *core_vdev) 135 + { 136 + struct xe_vfio_pci_core_device *xe_vdev = 137 + container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); 138 + struct vfio_pci_core_device *vdev = &xe_vdev->core_device; 139 + int ret; 140 + 141 + ret = vfio_pci_core_enable(vdev); 142 + if (ret) 143 + return ret; 144 + 145 + xe_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING; 146 + 147 + vfio_pci_core_finish_enable(vdev); 148 + 149 + return 0; 150 + } 151 + 152 + static void xe_vfio_pci_close_device(struct vfio_device *core_vdev) 153 + { 154 + struct xe_vfio_pci_core_device *xe_vdev = 155 + container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); 156 + 157 + xe_vfio_pci_state_mutex_lock(xe_vdev); 158 + xe_vfio_pci_reset(xe_vdev); 159 + xe_vfio_pci_state_mutex_unlock(xe_vdev); 160 + vfio_pci_core_close_device(core_vdev); 161 + } 162 + 163 + static int xe_vfio_pci_release_file(struct inode *inode, struct file *filp) 164 + { 165 + struct xe_vfio_pci_migration_file *migf = filp->private_data; 166 + 167 + mutex_destroy(&migf->lock); 168 + kfree(migf); 169 + 170 + return 0; 171 + } 172 + 173 + static ssize_t xe_vfio_pci_save_read(struct file *filp, char __user *buf, size_t len, loff_t *pos) 174 + { 175 + struct xe_vfio_pci_migration_file *migf = filp->private_data; 176 + ssize_t ret; 177 + 178 + if (pos) 179 + return -ESPIPE; 180 + 181 + mutex_lock(&migf->lock); 182 + if (migf->disabled) { 183 + mutex_unlock(&migf->lock); 184 + return -ENODEV; 185 + } 186 + 187 + ret = xe_sriov_vfio_data_read(migf->xe_vdev->xe, migf->xe_vdev->vfid, buf, len); 188 + mutex_unlock(&migf->lock); 189 + 190 + return ret; 191 + } 192 + 193 + static const struct file_operations xe_vfio_pci_save_fops = { 194 + .owner = THIS_MODULE, 195 + .read = xe_vfio_pci_save_read, 196 + .release = xe_vfio_pci_release_file, 197 + .llseek = noop_llseek, 198 + }; 199 + 200 + static ssize_t xe_vfio_pci_resume_write(struct file *filp, const char __user *buf, 201 + size_t len, loff_t *pos) 202 + { 203 + struct xe_vfio_pci_migration_file *migf = filp->private_data; 204 + ssize_t ret; 205 + 206 + if (pos) 207 + return -ESPIPE; 208 + 209 + mutex_lock(&migf->lock); 210 + if (migf->disabled) { 211 + mutex_unlock(&migf->lock); 212 + return -ENODEV; 213 + } 214 + 215 + ret = xe_sriov_vfio_data_write(migf->xe_vdev->xe, migf->xe_vdev->vfid, buf, len); 216 + mutex_unlock(&migf->lock); 217 + 218 + return ret; 219 + } 220 + 221 + static const struct file_operations xe_vfio_pci_resume_fops = { 222 + .owner = THIS_MODULE, 223 + .write = xe_vfio_pci_resume_write, 224 + .release = xe_vfio_pci_release_file, 225 + .llseek = noop_llseek, 226 + }; 227 + 228 + static const char *vfio_dev_state_str(u32 state) 229 + { 230 + switch (state) { 231 + case VFIO_DEVICE_STATE_RUNNING: return "running"; 232 + case VFIO_DEVICE_STATE_RUNNING_P2P: return "running_p2p"; 233 + case VFIO_DEVICE_STATE_STOP_COPY: return "stopcopy"; 234 + case VFIO_DEVICE_STATE_STOP: return "stop"; 235 + case VFIO_DEVICE_STATE_RESUMING: return "resuming"; 236 + case VFIO_DEVICE_STATE_ERROR: return "error"; 237 + default: return ""; 238 + } 239 + } 240 + 241 + enum xe_vfio_pci_file_type { 242 + XE_VFIO_FILE_SAVE = 0, 243 + XE_VFIO_FILE_RESUME, 244 + }; 245 + 246 + static struct xe_vfio_pci_migration_file * 247 + xe_vfio_pci_alloc_file(struct xe_vfio_pci_core_device *xe_vdev, 248 + enum xe_vfio_pci_file_type type) 249 + { 250 + struct xe_vfio_pci_migration_file *migf; 251 + const struct file_operations *fops; 252 + int flags; 253 + 254 + migf = kzalloc(sizeof(*migf), GFP_KERNEL_ACCOUNT); 255 + if (!migf) 256 + return ERR_PTR(-ENOMEM); 257 + 258 + fops = type == XE_VFIO_FILE_SAVE ? &xe_vfio_pci_save_fops : &xe_vfio_pci_resume_fops; 259 + flags = type == XE_VFIO_FILE_SAVE ? O_RDONLY : O_WRONLY; 260 + migf->filp = anon_inode_getfile("xe_vfio_mig", fops, migf, flags); 261 + if (IS_ERR(migf->filp)) { 262 + kfree(migf); 263 + return ERR_CAST(migf->filp); 264 + } 265 + 266 + mutex_init(&migf->lock); 267 + migf->xe_vdev = xe_vdev; 268 + xe_vdev->migf = migf; 269 + 270 + stream_open(migf->filp->f_inode, migf->filp); 271 + 272 + return migf; 273 + } 274 + 275 + static struct file * 276 + xe_vfio_set_state(struct xe_vfio_pci_core_device *xe_vdev, u32 new) 277 + { 278 + u32 cur = xe_vdev->mig_state; 279 + int ret; 280 + 281 + dev_dbg(xe_vdev_to_dev(xe_vdev), 282 + "state: %s->%s\n", vfio_dev_state_str(cur), vfio_dev_state_str(new)); 283 + 284 + /* 285 + * "STOP" handling is reused for "RUNNING_P2P", as the device doesn't 286 + * have the capability to selectively block outgoing p2p DMA transfers. 287 + * While the device is allowing BAR accesses when the VF is stopped, it 288 + * is not processing any new workload requests, effectively stopping 289 + * any outgoing DMA transfers (not just p2p). 290 + * Any VRAM / MMIO accesses occurring during "RUNNING_P2P" are kept and 291 + * will be migrated to target VF during stop-copy. 292 + */ 293 + if (cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_RUNNING_P2P) { 294 + ret = xe_sriov_vfio_suspend_device(xe_vdev->xe, xe_vdev->vfid); 295 + if (ret) 296 + goto err; 297 + 298 + return NULL; 299 + } 300 + 301 + if ((cur == VFIO_DEVICE_STATE_RUNNING_P2P && new == VFIO_DEVICE_STATE_STOP) || 302 + (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RUNNING_P2P)) 303 + return NULL; 304 + 305 + if (cur == VFIO_DEVICE_STATE_RUNNING_P2P && new == VFIO_DEVICE_STATE_RUNNING) { 306 + ret = xe_sriov_vfio_resume_device(xe_vdev->xe, xe_vdev->vfid); 307 + if (ret) 308 + goto err; 309 + 310 + return NULL; 311 + } 312 + 313 + if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_STOP_COPY) { 314 + struct xe_vfio_pci_migration_file *migf; 315 + 316 + migf = xe_vfio_pci_alloc_file(xe_vdev, XE_VFIO_FILE_SAVE); 317 + if (IS_ERR(migf)) { 318 + ret = PTR_ERR(migf); 319 + goto err; 320 + } 321 + get_file(migf->filp); 322 + 323 + ret = xe_sriov_vfio_stop_copy_enter(xe_vdev->xe, xe_vdev->vfid); 324 + if (ret) { 325 + fput(migf->filp); 326 + goto err; 327 + } 328 + 329 + return migf->filp; 330 + } 331 + 332 + if (cur == VFIO_DEVICE_STATE_STOP_COPY && new == VFIO_DEVICE_STATE_STOP) { 333 + if (xe_vdev->migf) 334 + xe_vfio_pci_put_file(xe_vdev); 335 + 336 + ret = xe_sriov_vfio_stop_copy_exit(xe_vdev->xe, xe_vdev->vfid); 337 + if (ret) 338 + goto err; 339 + 340 + return NULL; 341 + } 342 + 343 + if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RESUMING) { 344 + struct xe_vfio_pci_migration_file *migf; 345 + 346 + migf = xe_vfio_pci_alloc_file(xe_vdev, XE_VFIO_FILE_RESUME); 347 + if (IS_ERR(migf)) { 348 + ret = PTR_ERR(migf); 349 + goto err; 350 + } 351 + get_file(migf->filp); 352 + 353 + ret = xe_sriov_vfio_resume_data_enter(xe_vdev->xe, xe_vdev->vfid); 354 + if (ret) { 355 + fput(migf->filp); 356 + goto err; 357 + } 358 + 359 + return migf->filp; 360 + } 361 + 362 + if (cur == VFIO_DEVICE_STATE_RESUMING && new == VFIO_DEVICE_STATE_STOP) { 363 + if (xe_vdev->migf) 364 + xe_vfio_pci_put_file(xe_vdev); 365 + 366 + ret = xe_sriov_vfio_resume_data_exit(xe_vdev->xe, xe_vdev->vfid); 367 + if (ret) 368 + goto err; 369 + 370 + return NULL; 371 + } 372 + 373 + WARN(true, "Unknown state transition %d->%d", cur, new); 374 + return ERR_PTR(-EINVAL); 375 + 376 + err: 377 + dev_dbg(xe_vdev_to_dev(xe_vdev), 378 + "Failed to transition state: %s->%s err=%d\n", 379 + vfio_dev_state_str(cur), vfio_dev_state_str(new), ret); 380 + return ERR_PTR(ret); 381 + } 382 + 383 + static struct file * 384 + xe_vfio_pci_set_device_state(struct vfio_device *core_vdev, 385 + enum vfio_device_mig_state new_state) 386 + { 387 + struct xe_vfio_pci_core_device *xe_vdev = 388 + container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); 389 + enum vfio_device_mig_state next_state; 390 + struct file *f = NULL; 391 + int ret; 392 + 393 + xe_vfio_pci_state_mutex_lock(xe_vdev); 394 + while (new_state != xe_vdev->mig_state) { 395 + ret = vfio_mig_get_next_state(core_vdev, xe_vdev->mig_state, 396 + new_state, &next_state); 397 + if (ret) { 398 + xe_sriov_vfio_error(xe_vdev->xe, xe_vdev->vfid); 399 + f = ERR_PTR(ret); 400 + break; 401 + } 402 + f = xe_vfio_set_state(xe_vdev, next_state); 403 + if (IS_ERR(f)) 404 + break; 405 + 406 + xe_vdev->mig_state = next_state; 407 + 408 + /* Multiple state transitions with non-NULL file in the middle */ 409 + if (f && new_state != xe_vdev->mig_state) { 410 + fput(f); 411 + f = ERR_PTR(-EINVAL); 412 + break; 413 + } 414 + } 415 + xe_vfio_pci_state_mutex_unlock(xe_vdev); 416 + 417 + return f; 418 + } 419 + 420 + static int xe_vfio_pci_get_device_state(struct vfio_device *core_vdev, 421 + enum vfio_device_mig_state *curr_state) 422 + { 423 + struct xe_vfio_pci_core_device *xe_vdev = 424 + container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); 425 + 426 + xe_vfio_pci_state_mutex_lock(xe_vdev); 427 + *curr_state = xe_vdev->mig_state; 428 + xe_vfio_pci_state_mutex_unlock(xe_vdev); 429 + 430 + return 0; 431 + } 432 + 433 + static int xe_vfio_pci_get_data_size(struct vfio_device *vdev, 434 + unsigned long *stop_copy_length) 435 + { 436 + struct xe_vfio_pci_core_device *xe_vdev = 437 + container_of(vdev, struct xe_vfio_pci_core_device, core_device.vdev); 438 + 439 + xe_vfio_pci_state_mutex_lock(xe_vdev); 440 + *stop_copy_length = xe_sriov_vfio_stop_copy_size(xe_vdev->xe, xe_vdev->vfid); 441 + xe_vfio_pci_state_mutex_unlock(xe_vdev); 442 + 443 + return 0; 444 + } 445 + 446 + static const struct vfio_migration_ops xe_vfio_pci_migration_ops = { 447 + .migration_set_state = xe_vfio_pci_set_device_state, 448 + .migration_get_state = xe_vfio_pci_get_device_state, 449 + .migration_get_data_size = xe_vfio_pci_get_data_size, 450 + }; 451 + 452 + static void xe_vfio_pci_migration_init(struct xe_vfio_pci_core_device *xe_vdev) 453 + { 454 + struct vfio_device *core_vdev = &xe_vdev->core_device.vdev; 455 + struct pci_dev *pdev = to_pci_dev(core_vdev->dev); 456 + struct xe_device *xe = xe_sriov_vfio_get_pf(pdev); 457 + 458 + if (!xe) 459 + return; 460 + if (!xe_sriov_vfio_migration_supported(xe)) 461 + return; 462 + 463 + mutex_init(&xe_vdev->state_mutex); 464 + spin_lock_init(&xe_vdev->reset_lock); 465 + 466 + /* PF internal control uses vfid index starting from 1 */ 467 + xe_vdev->vfid = pci_iov_vf_id(pdev) + 1; 468 + xe_vdev->xe = xe; 469 + 470 + core_vdev->migration_flags = VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P; 471 + core_vdev->mig_ops = &xe_vfio_pci_migration_ops; 472 + } 473 + 474 + static void xe_vfio_pci_migration_fini(struct xe_vfio_pci_core_device *xe_vdev) 475 + { 476 + if (!xe_vdev->vfid) 477 + return; 478 + 479 + mutex_destroy(&xe_vdev->state_mutex); 480 + } 481 + 482 + static int xe_vfio_pci_init_dev(struct vfio_device *core_vdev) 483 + { 484 + struct xe_vfio_pci_core_device *xe_vdev = 485 + container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); 486 + 487 + xe_vfio_pci_migration_init(xe_vdev); 488 + 489 + return vfio_pci_core_init_dev(core_vdev); 490 + } 491 + 492 + static void xe_vfio_pci_release_dev(struct vfio_device *core_vdev) 493 + { 494 + struct xe_vfio_pci_core_device *xe_vdev = 495 + container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); 496 + 497 + xe_vfio_pci_migration_fini(xe_vdev); 498 + } 499 + 500 + static const struct vfio_device_ops xe_vfio_pci_ops = { 501 + .name = "xe-vfio-pci", 502 + .init = xe_vfio_pci_init_dev, 503 + .release = xe_vfio_pci_release_dev, 504 + .open_device = xe_vfio_pci_open_device, 505 + .close_device = xe_vfio_pci_close_device, 506 + .ioctl = vfio_pci_core_ioctl, 507 + .device_feature = vfio_pci_core_ioctl_feature, 508 + .read = vfio_pci_core_read, 509 + .write = vfio_pci_core_write, 510 + .mmap = vfio_pci_core_mmap, 511 + .request = vfio_pci_core_request, 512 + .match = vfio_pci_core_match, 513 + .match_token_uuid = vfio_pci_core_match_token_uuid, 514 + .bind_iommufd = vfio_iommufd_physical_bind, 515 + .unbind_iommufd = vfio_iommufd_physical_unbind, 516 + .attach_ioas = vfio_iommufd_physical_attach_ioas, 517 + .detach_ioas = vfio_iommufd_physical_detach_ioas, 518 + }; 519 + 520 + static int xe_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 521 + { 522 + struct xe_vfio_pci_core_device *xe_vdev; 523 + int ret; 524 + 525 + xe_vdev = vfio_alloc_device(xe_vfio_pci_core_device, core_device.vdev, &pdev->dev, 526 + &xe_vfio_pci_ops); 527 + if (IS_ERR(xe_vdev)) 528 + return PTR_ERR(xe_vdev); 529 + 530 + dev_set_drvdata(&pdev->dev, &xe_vdev->core_device); 531 + 532 + ret = vfio_pci_core_register_device(&xe_vdev->core_device); 533 + if (ret) { 534 + vfio_put_device(&xe_vdev->core_device.vdev); 535 + return ret; 536 + } 537 + 538 + return 0; 539 + } 540 + 541 + static void xe_vfio_pci_remove(struct pci_dev *pdev) 542 + { 543 + struct xe_vfio_pci_core_device *xe_vdev = pci_get_drvdata(pdev); 544 + 545 + vfio_pci_core_unregister_device(&xe_vdev->core_device); 546 + vfio_put_device(&xe_vdev->core_device.vdev); 547 + } 548 + 549 + #define INTEL_PCI_VFIO_DEVICE(_id) { \ 550 + PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_INTEL, (_id)) \ 551 + } 552 + 553 + static const struct pci_device_id xe_vfio_pci_table[] = { 554 + INTEL_PTL_IDS(INTEL_PCI_VFIO_DEVICE), 555 + INTEL_WCL_IDS(INTEL_PCI_VFIO_DEVICE), 556 + INTEL_BMG_IDS(INTEL_PCI_VFIO_DEVICE), 557 + {} 558 + }; 559 + MODULE_DEVICE_TABLE(pci, xe_vfio_pci_table); 560 + 561 + static struct pci_driver xe_vfio_pci_driver = { 562 + .name = "xe-vfio-pci", 563 + .id_table = xe_vfio_pci_table, 564 + .probe = xe_vfio_pci_probe, 565 + .remove = xe_vfio_pci_remove, 566 + .err_handler = &xe_vfio_pci_err_handlers, 567 + .driver_managed_dma = true, 568 + }; 569 + module_pci_driver(xe_vfio_pci_driver); 570 + 571 + MODULE_LICENSE("GPL"); 572 + MODULE_AUTHOR("Michał Winiarski <michal.winiarski@intel.com>"); 573 + MODULE_DESCRIPTION("VFIO PCI driver with migration support for Intel Graphics");

+143

include/drm/intel/xe_sriov_vfio.h

··· 1 + /* SPDX-License-Identifier: MIT */ 2 + /* 3 + * Copyright © 2025 Intel Corporation 4 + */ 5 + 6 + #ifndef _XE_SRIOV_VFIO_H_ 7 + #define _XE_SRIOV_VFIO_H_ 8 + 9 + #include <linux/types.h> 10 + 11 + struct pci_dev; 12 + struct xe_device; 13 + 14 + /** 15 + * xe_sriov_vfio_get_pf() - Get PF &xe_device. 16 + * @pdev: the VF &pci_dev device 17 + * 18 + * Return: pointer to PF &xe_device, NULL otherwise. 19 + */ 20 + struct xe_device *xe_sriov_vfio_get_pf(struct pci_dev *pdev); 21 + 22 + /** 23 + * xe_sriov_vfio_migration_supported() - Check if migration is supported. 24 + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() 25 + * 26 + * Return: true if migration is supported, false otherwise. 27 + */ 28 + bool xe_sriov_vfio_migration_supported(struct xe_device *xe); 29 + 30 + /** 31 + * xe_sriov_vfio_wait_flr_done() - Wait for VF FLR completion. 32 + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() 33 + * @vfid: the VF identifier (can't be 0) 34 + * 35 + * This function will wait until VF FLR is processed by PF on all tiles (or 36 + * until timeout occurs). 37 + * 38 + * Return: 0 on success or a negative error code on failure. 39 + */ 40 + int xe_sriov_vfio_wait_flr_done(struct xe_device *xe, unsigned int vfid); 41 + 42 + /** 43 + * xe_sriov_vfio_suspend_device() - Suspend VF. 44 + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() 45 + * @vfid: the VF identifier (can't be 0) 46 + * 47 + * This function will pause VF on all tiles/GTs. 48 + * 49 + * Return: 0 on success or a negative error code on failure. 50 + */ 51 + int xe_sriov_vfio_suspend_device(struct xe_device *xe, unsigned int vfid); 52 + 53 + /** 54 + * xe_sriov_vfio_resume_device() - Resume VF. 55 + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() 56 + * @vfid: the VF identifier (can't be 0) 57 + * 58 + * This function will resume VF on all tiles. 59 + * 60 + * Return: 0 on success or a negative error code on failure. 61 + */ 62 + int xe_sriov_vfio_resume_device(struct xe_device *xe, unsigned int vfid); 63 + 64 + /** 65 + * xe_sriov_vfio_stop_copy_enter() - Initiate a VF device migration data save. 66 + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() 67 + * @vfid: the VF identifier (can't be 0) 68 + * 69 + * Return: 0 on success or a negative error code on failure. 70 + */ 71 + int xe_sriov_vfio_stop_copy_enter(struct xe_device *xe, unsigned int vfid); 72 + 73 + /** 74 + * xe_sriov_vfio_stop_copy_exit() - Finish a VF device migration data save. 75 + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() 76 + * @vfid: the VF identifier (can't be 0) 77 + * 78 + * Return: 0 on success or a negative error code on failure. 79 + */ 80 + int xe_sriov_vfio_stop_copy_exit(struct xe_device *xe, unsigned int vfid); 81 + 82 + /** 83 + * xe_sriov_vfio_resume_data_enter() - Initiate a VF device migration data restore. 84 + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() 85 + * @vfid: the VF identifier (can't be 0) 86 + * 87 + * Return: 0 on success or a negative error code on failure. 88 + */ 89 + int xe_sriov_vfio_resume_data_enter(struct xe_device *xe, unsigned int vfid); 90 + 91 + /** 92 + * xe_sriov_vfio_resume_data_exit() - Finish a VF device migration data restore. 93 + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() 94 + * @vfid: the VF identifier (can't be 0) 95 + * 96 + * Return: 0 on success or a negative error code on failure. 97 + */ 98 + int xe_sriov_vfio_resume_data_exit(struct xe_device *xe, unsigned int vfid); 99 + 100 + /** 101 + * xe_sriov_vfio_error() - Move VF device to error state. 102 + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() 103 + * @vfid: the VF identifier (can't be 0) 104 + * 105 + * Reset is needed to move it out of error state. 106 + * 107 + * Return: 0 on success or a negative error code on failure. 108 + */ 109 + int xe_sriov_vfio_error(struct xe_device *xe, unsigned int vfid); 110 + 111 + /** 112 + * xe_sriov_vfio_data_read() - Read migration data from the VF device. 113 + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() 114 + * @vfid: the VF identifier (can't be 0) 115 + * @buf: start address of userspace buffer 116 + * @len: requested read size from userspace 117 + * 118 + * Return: number of bytes that has been successfully read, 119 + * 0 if no more migration data is available, -errno on failure. 120 + */ 121 + ssize_t xe_sriov_vfio_data_read(struct xe_device *xe, unsigned int vfid, 122 + char __user *buf, size_t len); 123 + /** 124 + * xe_sriov_vfio_data_write() - Write migration data to the VF device. 125 + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() 126 + * @vfid: the VF identifier (can't be 0) 127 + * @buf: start address of userspace buffer 128 + * @len: requested write size from userspace 129 + * 130 + * Return: number of bytes that has been successfully written, -errno on failure. 131 + */ 132 + ssize_t xe_sriov_vfio_data_write(struct xe_device *xe, unsigned int vfid, 133 + const char __user *buf, size_t len); 134 + /** 135 + * xe_sriov_vfio_stop_copy_size() - Get a size estimate of VF device migration data. 136 + * @xe: the PF &xe_device obtained by calling xe_sriov_vfio_get_pf() 137 + * @vfid: the VF identifier (can't be 0) 138 + * 139 + * Return: migration data size in bytes or a negative error code on failure. 140 + */ 141 + ssize_t xe_sriov_vfio_stop_copy_size(struct xe_device *xe, unsigned int vfid); 142 + 143 + #endif

Configure Feed

Configure Feed