mirror of OpenBSD xenocara tree
github.com/openbsd/xenocara
openbsd
1/*
2 * Copyright © 2014-2015 Broadcom
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24#ifndef NIR_BUILDER_H
25#define NIR_BUILDER_H
26
27#include "util/bitscan.h"
28#include "util/half_float.h"
29#include "nir_control_flow.h"
30
31#ifdef __cplusplus
32extern "C" {
33#endif
34
35struct exec_list;
36
37typedef struct nir_builder {
38 nir_cursor cursor;
39
40 /* Whether new ALU instructions will be marked "exact" */
41 bool exact;
42
43 /* Float_controls2 bits. See nir_alu_instr for details. */
44 uint32_t fp_fast_math;
45
46 nir_shader *shader;
47 nir_function_impl *impl;
48} nir_builder;
49
50static inline nir_builder
51nir_builder_create(nir_function_impl *impl)
52{
53 nir_builder b;
54 memset(&b, 0, sizeof(b));
55 b.exact = false;
56 b.impl = impl;
57 b.shader = impl->function->shader;
58 return b;
59}
60
61/* Requires the cursor to be inside a nir_function_impl. */
62static inline nir_builder
63nir_builder_at(nir_cursor cursor)
64{
65 nir_cf_node *current_block = &nir_cursor_current_block(cursor)->cf_node;
66
67 nir_builder b = nir_builder_create(nir_cf_node_get_function(current_block));
68 b.cursor = cursor;
69 return b;
70}
71
72nir_builder MUST_CHECK PRINTFLIKE(3, 4)
73 nir_builder_init_simple_shader(gl_shader_stage stage,
74 const nir_shader_compiler_options *options,
75 const char *name, ...);
76
77typedef bool (*nir_instr_pass_cb)(struct nir_builder *, nir_instr *, void *);
78typedef bool (*nir_intrinsic_pass_cb)(struct nir_builder *,
79 nir_intrinsic_instr *, void *);
80typedef bool (*nir_alu_pass_cb)(struct nir_builder *,
81 nir_alu_instr *, void *);
82
83/**
84 * Iterates over all the instructions in a NIR function and calls the given pass
85 * on them.
86 *
87 * The pass should return true if it modified the function. In that case, only
88 * the preserved metadata flags will be preserved in the function impl.
89 *
90 * The builder will be initialized to point at the function impl, but its
91 * cursor is unset.
92 */
93static inline bool
94nir_function_instructions_pass(nir_function_impl *impl,
95 nir_instr_pass_cb pass,
96 nir_metadata preserved,
97 void *cb_data)
98{
99 bool progress = false;
100 nir_builder b = nir_builder_create(impl);
101
102 nir_foreach_block_safe(block, impl) {
103 nir_foreach_instr_safe(instr, block) {
104 progress |= pass(&b, instr, cb_data);
105 }
106 }
107
108 if (progress) {
109 nir_metadata_preserve(impl, preserved);
110 } else {
111 nir_metadata_preserve(impl, nir_metadata_all);
112 }
113
114 return progress;
115}
116
117/**
118 * Iterates over all the instructions in a NIR shader and calls the given pass
119 * on them.
120 *
121 * The pass should return true if it modified the shader. In that case, only
122 * the preserved metadata flags will be preserved in the function impl.
123 *
124 * The builder will be initialized to point at the function impl, but its
125 * cursor is unset.
126 */
127static inline bool
128nir_shader_instructions_pass(nir_shader *shader,
129 nir_instr_pass_cb pass,
130 nir_metadata preserved,
131 void *cb_data)
132{
133 bool progress = false;
134
135 nir_foreach_function_impl(impl, shader) {
136 progress |= nir_function_instructions_pass(impl, pass,
137 preserved, cb_data);
138 }
139
140 return progress;
141}
142
143/**
144 * Iterates over all the intrinsics in a NIR shader and calls the given pass on
145 * them.
146 *
147 * The pass should return true if it modified the shader. In that case, only
148 * the preserved metadata flags will be preserved in the function impl.
149 *
150 * The builder will be initialized to point at the function impl, but its
151 * cursor is unset.
152 */
153static inline bool
154nir_shader_intrinsics_pass(nir_shader *shader,
155 nir_intrinsic_pass_cb pass,
156 nir_metadata preserved,
157 void *cb_data)
158{
159 bool progress = false;
160
161 nir_foreach_function_impl(impl, shader) {
162 bool func_progress = false;
163 nir_builder b = nir_builder_create(impl);
164
165 nir_foreach_block_safe(block, impl) {
166 nir_foreach_instr_safe(instr, block) {
167 if (instr->type == nir_instr_type_intrinsic) {
168 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
169 func_progress |= pass(&b, intr, cb_data);
170 }
171 }
172 }
173
174 if (func_progress) {
175 nir_metadata_preserve(impl, preserved);
176 progress = true;
177 } else {
178 nir_metadata_preserve(impl, nir_metadata_all);
179 }
180 }
181
182 return progress;
183}
184
185/* As above, but for ALU */
186static inline bool
187nir_shader_alu_pass(nir_shader *shader,
188 nir_alu_pass_cb pass,
189 nir_metadata preserved,
190 void *cb_data)
191{
192 bool progress = false;
193
194 nir_foreach_function_impl(impl, shader) {
195 bool func_progress = false;
196 nir_builder b = nir_builder_create(impl);
197
198 nir_foreach_block_safe(block, impl) {
199 nir_foreach_instr_safe(instr, block) {
200 if (instr->type == nir_instr_type_alu) {
201 nir_alu_instr *intr = nir_instr_as_alu(instr);
202 func_progress |= pass(&b, intr, cb_data);
203 }
204 }
205 }
206
207 if (func_progress) {
208 nir_metadata_preserve(impl, preserved);
209 progress = true;
210 } else {
211 nir_metadata_preserve(impl, nir_metadata_all);
212 }
213 }
214
215 return progress;
216}
217
218void nir_builder_instr_insert(nir_builder *build, nir_instr *instr);
219void nir_builder_instr_insert_at_top(nir_builder *build, nir_instr *instr);
220
221static inline nir_instr *
222nir_builder_last_instr(nir_builder *build)
223{
224 assert(build->cursor.option == nir_cursor_after_instr);
225 return build->cursor.instr;
226}
227
228/* General nir_build_alu() taking a variable arg count with NULLs for the rest. */
229nir_def *
230nir_build_alu(nir_builder *build, nir_op op, nir_def *src0,
231 nir_def *src1, nir_def *src2, nir_def *src3);
232
233/* Fixed-arg-count variants to reduce size of codegen. */
234nir_def *
235nir_build_alu1(nir_builder *build, nir_op op, nir_def *src0);
236nir_def *
237nir_build_alu2(nir_builder *build, nir_op op, nir_def *src0,
238 nir_def *src1);
239nir_def *
240nir_build_alu3(nir_builder *build, nir_op op, nir_def *src0,
241 nir_def *src1, nir_def *src2);
242nir_def *
243nir_build_alu4(nir_builder *build, nir_op op, nir_def *src0,
244 nir_def *src1, nir_def *src2, nir_def *src3);
245
246nir_def *nir_build_alu_src_arr(nir_builder *build, nir_op op, nir_def **srcs);
247
248nir_def *
249nir_build_tex_deref_instr(nir_builder *build, nir_texop op,
250 nir_deref_instr *texture,
251 nir_deref_instr *sampler,
252 unsigned num_extra_srcs,
253 const nir_tex_src *extra_srcs);
254
255nir_instr *nir_builder_last_instr(nir_builder *build);
256
257void nir_builder_cf_insert(nir_builder *build, nir_cf_node *cf);
258
259bool nir_builder_is_inside_cf(nir_builder *build, nir_cf_node *cf_node);
260
261nir_if *
262nir_push_if(nir_builder *build, nir_def *condition);
263
264nir_if *
265nir_push_else(nir_builder *build, nir_if *nif);
266
267void nir_pop_if(nir_builder *build, nir_if *nif);
268
269nir_def *
270nir_if_phi(nir_builder *build, nir_def *then_def, nir_def *else_def);
271
272nir_loop *
273nir_push_loop(nir_builder *build);
274
275nir_loop *
276nir_push_continue(nir_builder *build, nir_loop *loop);
277
278void nir_pop_loop(nir_builder *build, nir_loop *loop);
279
280static inline nir_def *
281nir_undef(nir_builder *build, unsigned num_components, unsigned bit_size)
282{
283 nir_undef_instr *undef =
284 nir_undef_instr_create(build->shader, num_components, bit_size);
285 if (!undef)
286 return NULL;
287
288 nir_builder_instr_insert_at_top(build, &undef->instr);
289
290 return &undef->def;
291}
292
293static inline nir_def *
294nir_build_imm(nir_builder *build, unsigned num_components,
295 unsigned bit_size, const nir_const_value *value)
296{
297 nir_load_const_instr *load_const =
298 nir_load_const_instr_create(build->shader, num_components, bit_size);
299 if (!load_const)
300 return NULL;
301
302 memcpy(load_const->value, value, sizeof(nir_const_value) * num_components);
303
304 nir_builder_instr_insert(build, &load_const->instr);
305
306 return &load_const->def;
307}
308
309static inline nir_def *
310nir_imm_zero(nir_builder *build, unsigned num_components, unsigned bit_size)
311{
312 nir_load_const_instr *load_const =
313 nir_load_const_instr_create(build->shader, num_components, bit_size);
314
315 /* nir_load_const_instr_create uses rzalloc so it's already zero */
316
317 nir_builder_instr_insert(build, &load_const->instr);
318
319 return &load_const->def;
320}
321
322static inline nir_def *
323nir_imm_boolN_t(nir_builder *build, bool x, unsigned bit_size)
324{
325 nir_const_value v = nir_const_value_for_bool(x, bit_size);
326 return nir_build_imm(build, 1, bit_size, &v);
327}
328
329static inline nir_def *
330nir_imm_bool(nir_builder *build, bool x)
331{
332 return nir_imm_boolN_t(build, x, 1);
333}
334
335static inline nir_def *
336nir_imm_true(nir_builder *build)
337{
338 return nir_imm_bool(build, true);
339}
340
341static inline nir_def *
342nir_imm_false(nir_builder *build)
343{
344 return nir_imm_bool(build, false);
345}
346
347static inline nir_def *
348nir_imm_floatN_t(nir_builder *build, double x, unsigned bit_size)
349{
350 nir_const_value v = nir_const_value_for_float(x, bit_size);
351 return nir_build_imm(build, 1, bit_size, &v);
352}
353
354static inline nir_def *
355nir_imm_float16(nir_builder *build, float x)
356{
357 return nir_imm_floatN_t(build, x, 16);
358}
359
360static inline nir_def *
361nir_imm_float(nir_builder *build, float x)
362{
363 return nir_imm_floatN_t(build, x, 32);
364}
365
366static inline nir_def *
367nir_imm_double(nir_builder *build, double x)
368{
369 return nir_imm_floatN_t(build, x, 64);
370}
371
372static inline nir_def *
373nir_imm_vec2(nir_builder *build, float x, float y)
374{
375 nir_const_value v[2] = {
376 nir_const_value_for_float(x, 32),
377 nir_const_value_for_float(y, 32),
378 };
379 return nir_build_imm(build, 2, 32, v);
380}
381
382static inline nir_def *
383nir_imm_vec3(nir_builder *build, float x, float y, float z)
384{
385 nir_const_value v[3] = {
386 nir_const_value_for_float(x, 32),
387 nir_const_value_for_float(y, 32),
388 nir_const_value_for_float(z, 32),
389 };
390 return nir_build_imm(build, 3, 32, v);
391}
392
393static inline nir_def *
394nir_imm_vec4(nir_builder *build, float x, float y, float z, float w)
395{
396 nir_const_value v[4] = {
397 nir_const_value_for_float(x, 32),
398 nir_const_value_for_float(y, 32),
399 nir_const_value_for_float(z, 32),
400 nir_const_value_for_float(w, 32),
401 };
402
403 return nir_build_imm(build, 4, 32, v);
404}
405
406static inline nir_def *
407nir_imm_vec4_16(nir_builder *build, float x, float y, float z, float w)
408{
409 nir_const_value v[4] = {
410 nir_const_value_for_float(x, 16),
411 nir_const_value_for_float(y, 16),
412 nir_const_value_for_float(z, 16),
413 nir_const_value_for_float(w, 16),
414 };
415
416 return nir_build_imm(build, 4, 16, v);
417}
418
419static inline nir_def *
420nir_imm_intN_t(nir_builder *build, uint64_t x, unsigned bit_size)
421{
422 nir_const_value v = nir_const_value_for_raw_uint(x, bit_size);
423 return nir_build_imm(build, 1, bit_size, &v);
424}
425
426static inline nir_def *
427nir_imm_int(nir_builder *build, int x)
428{
429 return nir_imm_intN_t(build, x, 32);
430}
431
432static inline nir_def *
433nir_imm_int64(nir_builder *build, int64_t x)
434{
435 return nir_imm_intN_t(build, x, 64);
436}
437
438static inline nir_def *
439nir_imm_ivec2(nir_builder *build, int x, int y)
440{
441 nir_const_value v[2] = {
442 nir_const_value_for_int(x, 32),
443 nir_const_value_for_int(y, 32),
444 };
445
446 return nir_build_imm(build, 2, 32, v);
447}
448
449static inline nir_def *
450nir_imm_ivec3_intN(nir_builder *build, int x, int y, int z, unsigned bit_size)
451{
452 nir_const_value v[3] = {
453 nir_const_value_for_int(x, bit_size),
454 nir_const_value_for_int(y, bit_size),
455 nir_const_value_for_int(z, bit_size),
456 };
457
458 return nir_build_imm(build, 3, bit_size, v);
459}
460
461static inline nir_def *
462nir_imm_uvec2_intN(nir_builder *build, unsigned x, unsigned y,
463 unsigned bit_size)
464{
465 nir_const_value v[2] = {
466 nir_const_value_for_uint(x, bit_size),
467 nir_const_value_for_uint(y, bit_size),
468 };
469
470 return nir_build_imm(build, 2, bit_size, v);
471}
472
473static inline nir_def *
474nir_imm_uvec3_intN(nir_builder *build, unsigned x, unsigned y, unsigned z,
475 unsigned bit_size)
476{
477 nir_const_value v[3] = {
478 nir_const_value_for_uint(x, bit_size),
479 nir_const_value_for_uint(y, bit_size),
480 nir_const_value_for_uint(z, bit_size),
481 };
482
483 return nir_build_imm(build, 3, bit_size, v);
484}
485
486static inline nir_def *
487nir_imm_ivec3(nir_builder *build, int x, int y, int z)
488{
489 return nir_imm_ivec3_intN(build, x, y, z, 32);
490}
491
492static inline nir_def *
493nir_imm_ivec4_intN(nir_builder *build, int x, int y, int z, int w,
494 unsigned bit_size)
495{
496 nir_const_value v[4] = {
497 nir_const_value_for_int(x, bit_size),
498 nir_const_value_for_int(y, bit_size),
499 nir_const_value_for_int(z, bit_size),
500 nir_const_value_for_int(w, bit_size),
501 };
502
503 return nir_build_imm(build, 4, bit_size, v);
504}
505
506static inline nir_def *
507nir_imm_ivec4(nir_builder *build, int x, int y, int z, int w)
508{
509 return nir_imm_ivec4_intN(build, x, y, z, w, 32);
510}
511
512nir_def *
513nir_builder_alu_instr_finish_and_insert(nir_builder *build, nir_alu_instr *instr);
514
515/* for the couple special cases with more than 4 src args: */
516nir_def *
517nir_build_alu_src_arr(nir_builder *build, nir_op op, nir_def **srcs);
518
519/* Generic builder for system values. */
520nir_def *
521nir_load_system_value(nir_builder *build, nir_intrinsic_op op, int index,
522 unsigned num_components, unsigned bit_size);
523
524#include "nir_builder_opcodes.h"
525#undef nir_deref_mode_is
526
527nir_def *
528nir_type_convert(nir_builder *b,
529 nir_def *src,
530 nir_alu_type src_type,
531 nir_alu_type dest_type,
532 nir_rounding_mode rnd);
533
534static inline nir_def *
535nir_convert_to_bit_size(nir_builder *b,
536 nir_def *src,
537 nir_alu_type type,
538 unsigned bit_size)
539{
540 return nir_type_convert(b, src, type, (nir_alu_type)(type | bit_size),
541 nir_rounding_mode_undef);
542}
543
544static inline nir_def *
545nir_i2iN(nir_builder *b, nir_def *src, unsigned bit_size)
546{
547 return nir_convert_to_bit_size(b, src, nir_type_int, bit_size);
548}
549
550static inline nir_def *
551nir_u2uN(nir_builder *b, nir_def *src, unsigned bit_size)
552{
553 return nir_convert_to_bit_size(b, src, nir_type_uint, bit_size);
554}
555
556static inline nir_def *
557nir_b2bN(nir_builder *b, nir_def *src, unsigned bit_size)
558{
559 return nir_convert_to_bit_size(b, src, nir_type_bool, bit_size);
560}
561
562static inline nir_def *
563nir_f2fN(nir_builder *b, nir_def *src, unsigned bit_size)
564{
565 return nir_convert_to_bit_size(b, src, nir_type_float, bit_size);
566}
567
568static inline nir_def *
569nir_i2b(nir_builder *b, nir_def *src)
570{
571 return nir_ine_imm(b, src, 0);
572}
573
574static inline nir_def *
575nir_b2iN(nir_builder *b, nir_def *src, uint32_t bit_size)
576{
577 return nir_type_convert(b, src, nir_type_bool,
578 (nir_alu_type)(nir_type_int | bit_size),
579 nir_rounding_mode_undef);
580}
581
582static inline nir_def *
583nir_b2fN(nir_builder *b, nir_def *src, uint32_t bit_size)
584{
585 return nir_type_convert(b, src, nir_type_bool,
586 (nir_alu_type)(nir_type_float | bit_size),
587 nir_rounding_mode_undef);
588}
589
590static inline nir_def *
591nir_i2fN(nir_builder *b, nir_def *src, unsigned bit_size)
592{
593 return nir_type_convert(b, src, nir_type_int,
594 (nir_alu_type)(nir_type_float | bit_size),
595 nir_rounding_mode_undef);
596}
597
598static inline nir_def *
599nir_u2fN(nir_builder *b, nir_def *src, unsigned bit_size)
600{
601 return nir_type_convert(b, src, nir_type_uint,
602 (nir_alu_type)(nir_type_float | bit_size),
603 nir_rounding_mode_undef);
604}
605
606static inline nir_def *
607nir_f2uN(nir_builder *b, nir_def *src, unsigned bit_size)
608{
609 return nir_type_convert(b, src, nir_type_float,
610 (nir_alu_type)(nir_type_uint | bit_size),
611 nir_rounding_mode_undef);
612}
613
614static inline nir_def *
615nir_f2iN(nir_builder *b, nir_def *src, unsigned bit_size)
616{
617 return nir_type_convert(b, src, nir_type_float,
618 (nir_alu_type)(nir_type_int | bit_size),
619 nir_rounding_mode_undef);
620}
621
622static inline nir_def *
623nir_vec(nir_builder *build, nir_def **comp, unsigned num_components)
624{
625 return nir_build_alu_src_arr(build, nir_op_vec(num_components), comp);
626}
627
628nir_def *
629nir_vec_scalars(nir_builder *build, nir_scalar *comp, unsigned num_components);
630
631static inline nir_def *
632nir_mov_alu(nir_builder *build, nir_alu_src src, unsigned num_components)
633{
634 if (src.src.ssa->num_components == num_components) {
635 bool any_swizzles = false;
636 for (unsigned i = 0; i < num_components; i++) {
637 if (src.swizzle[i] != i)
638 any_swizzles = true;
639 }
640 if (!any_swizzles)
641 return src.src.ssa;
642 }
643
644 nir_alu_instr *mov = nir_alu_instr_create(build->shader, nir_op_mov);
645 nir_def_init(&mov->instr, &mov->def, num_components,
646 nir_src_bit_size(src.src));
647 mov->exact = build->exact;
648 mov->fp_fast_math = build->fp_fast_math;
649 mov->src[0] = src;
650 nir_builder_instr_insert(build, &mov->instr);
651
652 return &mov->def;
653}
654
655/**
656 * Construct a mov that reswizzles the source's components.
657 */
658static inline nir_def *
659nir_swizzle(nir_builder *build, nir_def *src, const unsigned *swiz,
660 unsigned num_components)
661{
662 assert(num_components <= NIR_MAX_VEC_COMPONENTS);
663 nir_alu_src alu_src = { NIR_SRC_INIT };
664 alu_src.src = nir_src_for_ssa(src);
665
666 bool is_identity_swizzle = true;
667 for (unsigned i = 0; i < num_components && i < NIR_MAX_VEC_COMPONENTS; i++) {
668 if (swiz[i] != i)
669 is_identity_swizzle = false;
670 alu_src.swizzle[i] = (uint8_t)swiz[i];
671 }
672
673 if (num_components == src->num_components && is_identity_swizzle)
674 return src;
675
676 return nir_mov_alu(build, alu_src, num_components);
677}
678
679/* Selects the right fdot given the number of components in each source. */
680static inline nir_def *
681nir_fdot(nir_builder *build, nir_def *src0, nir_def *src1)
682{
683 assert(src0->num_components == src1->num_components);
684 switch (src0->num_components) {
685 case 1:
686 return nir_fmul(build, src0, src1);
687 case 2:
688 return nir_fdot2(build, src0, src1);
689 case 3:
690 return nir_fdot3(build, src0, src1);
691 case 4:
692 return nir_fdot4(build, src0, src1);
693 case 5:
694 return nir_fdot5(build, src0, src1);
695 case 8:
696 return nir_fdot8(build, src0, src1);
697 case 16:
698 return nir_fdot16(build, src0, src1);
699 default:
700 unreachable("bad component size");
701 }
702
703 return NULL;
704}
705
706static inline nir_def *
707nir_ball_iequal(nir_builder *b, nir_def *src0, nir_def *src1)
708{
709 switch (src0->num_components) {
710 case 1:
711 return nir_ieq(b, src0, src1);
712 case 2:
713 return nir_ball_iequal2(b, src0, src1);
714 case 3:
715 return nir_ball_iequal3(b, src0, src1);
716 case 4:
717 return nir_ball_iequal4(b, src0, src1);
718 case 5:
719 return nir_ball_iequal5(b, src0, src1);
720 case 8:
721 return nir_ball_iequal8(b, src0, src1);
722 case 16:
723 return nir_ball_iequal16(b, src0, src1);
724 default:
725 unreachable("bad component size");
726 }
727}
728
729static inline nir_def *
730nir_ball(nir_builder *b, nir_def *src)
731{
732 return nir_ball_iequal(b, src, nir_imm_true(b));
733}
734
735static inline nir_def *
736nir_bany_inequal(nir_builder *b, nir_def *src0, nir_def *src1)
737{
738 switch (src0->num_components) {
739 case 1:
740 return nir_ine(b, src0, src1);
741 case 2:
742 return nir_bany_inequal2(b, src0, src1);
743 case 3:
744 return nir_bany_inequal3(b, src0, src1);
745 case 4:
746 return nir_bany_inequal4(b, src0, src1);
747 case 5:
748 return nir_bany_inequal5(b, src0, src1);
749 case 8:
750 return nir_bany_inequal8(b, src0, src1);
751 case 16:
752 return nir_bany_inequal16(b, src0, src1);
753 default:
754 unreachable("bad component size");
755 }
756}
757
758static inline nir_def *
759nir_bany(nir_builder *b, nir_def *src)
760{
761 return nir_bany_inequal(b, src, nir_imm_false(b));
762}
763
764static inline nir_def *
765nir_channel(nir_builder *b, nir_def *def, unsigned c)
766{
767 return nir_swizzle(b, def, &c, 1);
768}
769
770static inline nir_def *
771nir_channel_or_undef(nir_builder *b, nir_def *def, signed int channel)
772{
773 if (channel >= 0 && channel < def->num_components)
774 return nir_channel(b, def, channel);
775 else
776 return nir_undef(b, 1, def->bit_size);
777}
778
779static inline nir_def *
780nir_channels(nir_builder *b, nir_def *def, nir_component_mask_t mask)
781{
782 unsigned num_channels = 0, swizzle[NIR_MAX_VEC_COMPONENTS] = { 0 };
783
784 for (unsigned i = 0; i < NIR_MAX_VEC_COMPONENTS; i++) {
785 if ((mask & (1 << i)) == 0)
786 continue;
787 swizzle[num_channels++] = i;
788 }
789
790 return nir_swizzle(b, def, swizzle, num_channels);
791}
792
793static inline nir_def *
794_nir_select_from_array_helper(nir_builder *b, nir_def **arr,
795 nir_def *idx,
796 unsigned start, unsigned end)
797{
798 if (start == end - 1) {
799 return arr[start];
800 } else {
801 unsigned mid = start + (end - start) / 2;
802 return nir_bcsel(b, nir_ilt_imm(b, idx, mid),
803 _nir_select_from_array_helper(b, arr, idx, start, mid),
804 _nir_select_from_array_helper(b, arr, idx, mid, end));
805 }
806}
807
808static inline nir_def *
809nir_select_from_ssa_def_array(nir_builder *b, nir_def **arr,
810 unsigned arr_len, nir_def *idx)
811{
812 return _nir_select_from_array_helper(b, arr, idx, 0, arr_len);
813}
814
815static inline nir_def *
816nir_vector_extract(nir_builder *b, nir_def *vec, nir_def *c)
817{
818 nir_src c_src = nir_src_for_ssa(c);
819 if (nir_src_is_const(c_src)) {
820 uint64_t c_const = nir_src_as_uint(c_src);
821 if (c_const < vec->num_components)
822 return nir_channel(b, vec, (unsigned)c_const);
823 else
824 return nir_undef(b, 1, vec->bit_size);
825 } else {
826 nir_def *comps[NIR_MAX_VEC_COMPONENTS];
827 for (unsigned i = 0; i < vec->num_components; i++)
828 comps[i] = nir_channel(b, vec, i);
829 return nir_select_from_ssa_def_array(b, comps, vec->num_components, c);
830 }
831}
832
833/** Replaces the component of `vec` specified by `c` with `scalar` */
834static inline nir_def *
835nir_vector_insert_imm(nir_builder *b, nir_def *vec,
836 nir_def *scalar, unsigned c)
837{
838 assert(scalar->num_components == 1);
839 assert(c < vec->num_components);
840
841 nir_op vec_op = nir_op_vec(vec->num_components);
842 nir_alu_instr *vec_instr = nir_alu_instr_create(b->shader, vec_op);
843
844 for (unsigned i = 0; i < vec->num_components; i++) {
845 if (i == c) {
846 vec_instr->src[i].src = nir_src_for_ssa(scalar);
847 vec_instr->src[i].swizzle[0] = 0;
848 } else {
849 vec_instr->src[i].src = nir_src_for_ssa(vec);
850 vec_instr->src[i].swizzle[0] = (uint8_t)i;
851 }
852 }
853
854 return nir_builder_alu_instr_finish_and_insert(b, vec_instr);
855}
856
857/** Replaces the component of `vec` specified by `c` with `scalar` */
858static inline nir_def *
859nir_vector_insert(nir_builder *b, nir_def *vec, nir_def *scalar,
860 nir_def *c)
861{
862 assert(scalar->num_components == 1);
863 assert(c->num_components == 1);
864
865 nir_src c_src = nir_src_for_ssa(c);
866 if (nir_src_is_const(c_src)) {
867 uint64_t c_const = nir_src_as_uint(c_src);
868 if (c_const < vec->num_components)
869 return nir_vector_insert_imm(b, vec, scalar, (unsigned )c_const);
870 else
871 return vec;
872 } else {
873 nir_const_value per_comp_idx_const[NIR_MAX_VEC_COMPONENTS];
874 for (unsigned i = 0; i < NIR_MAX_VEC_COMPONENTS; i++)
875 per_comp_idx_const[i] = nir_const_value_for_int(i, c->bit_size);
876 nir_def *per_comp_idx =
877 nir_build_imm(b, vec->num_components,
878 c->bit_size, per_comp_idx_const);
879
880 /* nir_builder will automatically splat out scalars to vectors so an
881 * insert is as simple as "if I'm the channel, replace me with the
882 * scalar."
883 */
884 return nir_bcsel(b, nir_ieq(b, c, per_comp_idx), scalar, vec);
885 }
886}
887
888static inline nir_def *
889nir_replicate(nir_builder *b, nir_def *scalar, unsigned num_components)
890{
891 assert(scalar->num_components == 1);
892 assert(num_components <= NIR_MAX_VEC_COMPONENTS);
893
894 nir_def *copies[NIR_MAX_VEC_COMPONENTS] = { NULL };
895 for (unsigned i = 0; i < num_components; ++i)
896 copies[i] = scalar;
897
898 return nir_vec(b, copies, num_components);
899}
900
901static inline nir_def *
902nir_iadd_imm(nir_builder *build, nir_def *x, uint64_t y)
903{
904 assert(x->bit_size <= 64);
905 y &= BITFIELD64_MASK(x->bit_size);
906
907 if (y == 0) {
908 return x;
909 } else {
910 return nir_iadd(build, x, nir_imm_intN_t(build, y, x->bit_size));
911 }
912}
913
914static inline nir_def *
915nir_iadd_imm_nuw(nir_builder *b, nir_def *x, uint64_t y)
916{
917 nir_def *d = nir_iadd_imm(b, x, y);
918 if (d != x && d->parent_instr->type == nir_instr_type_alu)
919 nir_instr_as_alu(d->parent_instr)->no_unsigned_wrap = true;
920 return d;
921}
922
923static inline nir_def *
924nir_iadd_nuw(nir_builder *b, nir_def *x, nir_def *y)
925{
926 nir_def *d = nir_iadd(b, x, y);
927 nir_instr_as_alu(d->parent_instr)->no_unsigned_wrap = true;
928 return d;
929}
930
931static inline nir_def *
932nir_fgt_imm(nir_builder *build, nir_def *src1, double src2)
933{
934 return nir_flt(build, nir_imm_floatN_t(build, src2, src1->bit_size), src1);
935}
936
937static inline nir_def *
938nir_fle_imm(nir_builder *build, nir_def *src1, double src2)
939{
940 return nir_fge(build, nir_imm_floatN_t(build, src2, src1->bit_size), src1);
941}
942
943/* Use nir_iadd(x, -y) for reversing parameter ordering */
944static inline nir_def *
945nir_isub_imm(nir_builder *build, uint64_t y, nir_def *x)
946{
947 return nir_isub(build, nir_imm_intN_t(build, y, x->bit_size), x);
948}
949
950static inline nir_def *
951nir_imax_imm(nir_builder *build, nir_def *x, int64_t y)
952{
953 return nir_imax(build, x, nir_imm_intN_t(build, y, x->bit_size));
954}
955
956static inline nir_def *
957nir_imin_imm(nir_builder *build, nir_def *x, int64_t y)
958{
959 return nir_imin(build, x, nir_imm_intN_t(build, y, x->bit_size));
960}
961
962static inline nir_def *
963nir_umax_imm(nir_builder *build, nir_def *x, uint64_t y)
964{
965 return nir_umax(build, x, nir_imm_intN_t(build, y, x->bit_size));
966}
967
968static inline nir_def *
969nir_umin_imm(nir_builder *build, nir_def *x, uint64_t y)
970{
971 return nir_umin(build, x, nir_imm_intN_t(build, y, x->bit_size));
972}
973
974static inline nir_def *
975_nir_mul_imm(nir_builder *build, nir_def *x, uint64_t y, bool amul)
976{
977 assert(x->bit_size <= 64);
978 y &= BITFIELD64_MASK(x->bit_size);
979
980 if (y == 0) {
981 return nir_imm_intN_t(build, 0, x->bit_size);
982 } else if (y == 1) {
983 return x;
984 } else if ((!build->shader->options ||
985 !build->shader->options->lower_bitops) &&
986 !(amul && (!build->shader->options ||
987 build->shader->options->has_amul)) &&
988 util_is_power_of_two_or_zero64(y)) {
989 return nir_ishl(build, x, nir_imm_int(build, ffsll(y) - 1));
990 } else if (amul) {
991 return nir_amul(build, x, nir_imm_intN_t(build, y, x->bit_size));
992 } else {
993 return nir_imul(build, x, nir_imm_intN_t(build, y, x->bit_size));
994 }
995}
996
997static inline nir_def *
998nir_imul_imm(nir_builder *build, nir_def *x, uint64_t y)
999{
1000 return _nir_mul_imm(build, x, y, false);
1001}
1002
1003static inline nir_def *
1004nir_amul_imm(nir_builder *build, nir_def *x, uint64_t y)
1005{
1006 return _nir_mul_imm(build, x, y, true);
1007}
1008
1009static inline nir_def *
1010nir_fadd_imm(nir_builder *build, nir_def *x, double y)
1011{
1012 return nir_fadd(build, x, nir_imm_floatN_t(build, y, x->bit_size));
1013}
1014
1015static inline nir_def *
1016nir_fsub_imm(nir_builder *build, double x, nir_def *y)
1017{
1018 return nir_fsub(build, nir_imm_floatN_t(build, x, y->bit_size), y);
1019}
1020
1021static inline nir_def *
1022nir_fmul_imm(nir_builder *build, nir_def *x, double y)
1023{
1024 return nir_fmul(build, x, nir_imm_floatN_t(build, y, x->bit_size));
1025}
1026
1027static inline nir_def *
1028nir_fdiv_imm(nir_builder *build, nir_def *x, double y)
1029{
1030 return nir_fdiv(build, x, nir_imm_floatN_t(build, y, x->bit_size));
1031}
1032
1033static inline nir_def *
1034nir_fpow_imm(nir_builder *build, nir_def *x, double y)
1035{
1036 return nir_fpow(build, x, nir_imm_floatN_t(build, y, x->bit_size));
1037}
1038
1039static inline nir_def *
1040nir_iand_imm(nir_builder *build, nir_def *x, uint64_t y)
1041{
1042 assert(x->bit_size <= 64);
1043 y &= BITFIELD64_MASK(x->bit_size);
1044
1045 if (y == 0) {
1046 return nir_imm_intN_t(build, 0, x->bit_size);
1047 } else if (y == BITFIELD64_MASK(x->bit_size)) {
1048 return x;
1049 } else {
1050 return nir_iand(build, x, nir_imm_intN_t(build, y, x->bit_size));
1051 }
1052}
1053
1054static inline nir_def *
1055nir_test_mask(nir_builder *build, nir_def *x, uint64_t mask)
1056{
1057 assert(mask <= BITFIELD64_MASK(x->bit_size));
1058 return nir_ine_imm(build, nir_iand_imm(build, x, mask), 0);
1059}
1060
1061static inline nir_def *
1062nir_ior_imm(nir_builder *build, nir_def *x, uint64_t y)
1063{
1064 assert(x->bit_size <= 64);
1065 y &= BITFIELD64_MASK(x->bit_size);
1066
1067 if (y == 0) {
1068 return x;
1069 } else if (y == BITFIELD64_MASK(x->bit_size)) {
1070 return nir_imm_intN_t(build, y, x->bit_size);
1071 } else
1072 return nir_ior(build, x, nir_imm_intN_t(build, y, x->bit_size));
1073}
1074
1075static inline nir_def *
1076nir_ishl_imm(nir_builder *build, nir_def *x, uint32_t y)
1077{
1078 if (y == 0) {
1079 return x;
1080 } else {
1081 assert(y < x->bit_size);
1082 return nir_ishl(build, x, nir_imm_int(build, y));
1083 }
1084}
1085
1086static inline nir_def *
1087nir_ishr_imm(nir_builder *build, nir_def *x, uint32_t y)
1088{
1089 if (y == 0) {
1090 return x;
1091 } else {
1092 return nir_ishr(build, x, nir_imm_int(build, y));
1093 }
1094}
1095
1096static inline nir_def *
1097nir_ushr_imm(nir_builder *build, nir_def *x, uint32_t y)
1098{
1099 if (y == 0) {
1100 return x;
1101 } else {
1102 return nir_ushr(build, x, nir_imm_int(build, y));
1103 }
1104}
1105
1106static inline nir_def *
1107nir_imod_imm(nir_builder *build, nir_def *x, uint64_t y)
1108{
1109 return nir_imod(build, x, nir_imm_intN_t(build, y, x->bit_size));
1110}
1111
1112static inline nir_def *
1113nir_udiv_imm(nir_builder *build, nir_def *x, uint64_t y)
1114{
1115 assert(x->bit_size <= 64);
1116 y &= BITFIELD64_MASK(x->bit_size);
1117
1118 if (y == 1) {
1119 return x;
1120 } else if (util_is_power_of_two_nonzero64(y)) {
1121 return nir_ushr_imm(build, x, ffsll(y) - 1);
1122 } else {
1123 return nir_udiv(build, x, nir_imm_intN_t(build, y, x->bit_size));
1124 }
1125}
1126
1127static inline nir_def *
1128nir_umod_imm(nir_builder *build, nir_def *x, uint64_t y)
1129{
1130 assert(y > 0 && y <= u_uintN_max(x->bit_size));
1131
1132 if (util_is_power_of_two_nonzero64(y)) {
1133 return nir_iand_imm(build, x, y - 1);
1134 } else {
1135 return nir_umod(build, x, nir_imm_intN_t(build, y, x->bit_size));
1136 }
1137}
1138
1139static inline nir_def *
1140nir_ibfe_imm(nir_builder *build, nir_def *x, uint32_t offset, uint32_t size)
1141{
1142 return nir_ibfe(build, x, nir_imm_int(build, offset), nir_imm_int(build, size));
1143}
1144
1145static inline nir_def *
1146nir_ubfe_imm(nir_builder *build, nir_def *x, uint32_t offset, uint32_t size)
1147{
1148 return nir_ubfe(build, x, nir_imm_int(build, offset), nir_imm_int(build, size));
1149}
1150
1151static inline nir_def *
1152nir_ubitfield_extract_imm(nir_builder *build, nir_def *x, uint32_t offset, uint32_t size)
1153{
1154 return nir_ubitfield_extract(build, x, nir_imm_int(build, offset), nir_imm_int(build, size));
1155}
1156
1157static inline nir_def *
1158nir_extract_u8_imm(nir_builder *b, nir_def *a, unsigned i)
1159{
1160 return nir_extract_u8(b, a, nir_imm_intN_t(b, i, a->bit_size));
1161}
1162
1163static inline nir_def *
1164nir_extract_i8_imm(nir_builder *b, nir_def *a, unsigned i)
1165{
1166 return nir_extract_i8(b, a, nir_imm_intN_t(b, i, a->bit_size));
1167}
1168
1169static inline nir_def *
1170nir_fclamp(nir_builder *b,
1171 nir_def *x, nir_def *min_val, nir_def *max_val)
1172{
1173 return nir_fmin(b, nir_fmax(b, x, min_val), max_val);
1174}
1175
1176static inline nir_def *
1177nir_iclamp(nir_builder *b,
1178 nir_def *x, nir_def *min_val, nir_def *max_val)
1179{
1180 return nir_imin(b, nir_imax(b, x, min_val), max_val);
1181}
1182
1183static inline nir_def *
1184nir_uclamp(nir_builder *b,
1185 nir_def *x, nir_def *min_val, nir_def *max_val)
1186{
1187 return nir_umin(b, nir_umax(b, x, min_val), max_val);
1188}
1189
1190static inline nir_def *
1191nir_ffma_imm12(nir_builder *build, nir_def *src0, double src1, double src2)
1192{
1193 if (build->shader->options &&
1194 build->shader->options->avoid_ternary_with_two_constants)
1195 return nir_fadd_imm(build, nir_fmul_imm(build, src0, src1), src2);
1196 else
1197 return nir_ffma(build, src0, nir_imm_floatN_t(build, src1, src0->bit_size),
1198 nir_imm_floatN_t(build, src2, src0->bit_size));
1199}
1200
1201static inline nir_def *
1202nir_ffma_imm1(nir_builder *build, nir_def *src0, double src1, nir_def *src2)
1203{
1204 return nir_ffma(build, src0, nir_imm_floatN_t(build, src1, src0->bit_size), src2);
1205}
1206
1207static inline nir_def *
1208nir_ffma_imm2(nir_builder *build, nir_def *src0, nir_def *src1, double src2)
1209{
1210 return nir_ffma(build, src0, src1, nir_imm_floatN_t(build, src2, src0->bit_size));
1211}
1212
1213static inline nir_def *
1214nir_a_minus_bc(nir_builder *build, nir_def *src0, nir_def *src1,
1215 nir_def *src2)
1216{
1217 return nir_ffma(build, nir_fneg(build, src1), src2, src0);
1218}
1219
1220static inline nir_def *
1221nir_pack_bits(nir_builder *b, nir_def *src, unsigned dest_bit_size)
1222{
1223 assert((unsigned)(src->num_components * src->bit_size) == dest_bit_size);
1224
1225 switch (dest_bit_size) {
1226 case 64:
1227 switch (src->bit_size) {
1228 case 32:
1229 return nir_pack_64_2x32(b, src);
1230 case 16:
1231 return nir_pack_64_4x16(b, src);
1232 case 8: {
1233 nir_def *lo = nir_pack_32_4x8(b, nir_channels(b, src, 0x0f));
1234 nir_def *hi = nir_pack_32_4x8(b, nir_channels(b, src, 0xf0));
1235 return nir_pack_64_2x32(b, nir_vec2(b, lo, hi));
1236 }
1237 default:
1238 break;
1239 }
1240 break;
1241
1242 case 32:
1243 switch (src->bit_size) {
1244 case 32: return src;
1245 case 16: return nir_pack_32_2x16(b, src);
1246 case 8: return nir_pack_32_4x8(b, src);
1247 default: break;
1248 }
1249
1250 break;
1251
1252 default:
1253 break;
1254 }
1255
1256 /* If we got here, we have no dedicated unpack opcode. */
1257 nir_def *dest = nir_imm_intN_t(b, 0, dest_bit_size);
1258 for (unsigned i = 0; i < src->num_components; i++) {
1259 nir_def *val = nir_u2uN(b, nir_channel(b, src, i), dest_bit_size);
1260 val = nir_ishl(b, val, nir_imm_int(b, i * src->bit_size));
1261 dest = nir_ior(b, dest, val);
1262 }
1263 return dest;
1264}
1265
1266static inline nir_def *
1267nir_unpack_bits(nir_builder *b, nir_def *src, unsigned dest_bit_size)
1268{
1269 assert(src->num_components == 1);
1270 assert(src->bit_size >= dest_bit_size);
1271 const unsigned dest_num_components = src->bit_size / dest_bit_size;
1272 assert(dest_num_components <= NIR_MAX_VEC_COMPONENTS);
1273
1274 switch (src->bit_size) {
1275 case 64:
1276 switch (dest_bit_size) {
1277 case 32:
1278 return nir_unpack_64_2x32(b, src);
1279 case 16:
1280 return nir_unpack_64_4x16(b, src);
1281 case 8: {
1282 nir_def *split = nir_unpack_64_2x32(b, src);
1283 nir_def *lo = nir_unpack_32_4x8(b, nir_channel(b, split, 0));
1284 nir_def *hi = nir_unpack_32_4x8(b, nir_channel(b, split, 1));
1285 return nir_vec8(b, nir_channel(b, lo, 0), nir_channel(b, lo, 1),
1286 nir_channel(b, lo, 2), nir_channel(b, lo, 3),
1287 nir_channel(b, hi, 0), nir_channel(b, hi, 1),
1288 nir_channel(b, hi, 2), nir_channel(b, hi, 3));
1289 }
1290 default:
1291 break;
1292 }
1293 break;
1294
1295 case 32:
1296 switch (dest_bit_size) {
1297 case 32: return src;
1298 case 16: return nir_unpack_32_2x16(b, src);
1299 case 8: return nir_unpack_32_4x8(b, src);
1300 default: break;
1301 }
1302
1303 break;
1304
1305 default:
1306 break;
1307 }
1308
1309 /* If we got here, we have no dedicated unpack opcode. */
1310 nir_def *dest_comps[NIR_MAX_VEC_COMPONENTS];
1311 for (unsigned i = 0; i < dest_num_components; i++) {
1312 nir_def *val = nir_ushr_imm(b, src, i * dest_bit_size);
1313 dest_comps[i] = nir_u2uN(b, val, dest_bit_size);
1314 }
1315 return nir_vec(b, dest_comps, dest_num_components);
1316}
1317
1318/**
1319 * Treats srcs as if it's one big blob of bits and extracts the range of bits
1320 * given by
1321 *
1322 * [first_bit, first_bit + dest_num_components * dest_bit_size)
1323 *
1324 * The range can have any alignment or size as long as it's an integer number
1325 * of destination components and fits inside the concatenated sources.
1326 *
1327 * TODO: The one caveat here is that we can't handle byte alignment if 64-bit
1328 * values are involved because that would require pack/unpack to/from a vec8
1329 * which NIR currently does not support.
1330 */
1331static inline nir_def *
1332nir_extract_bits(nir_builder *b, nir_def **srcs, unsigned num_srcs,
1333 unsigned first_bit,
1334 unsigned dest_num_components, unsigned dest_bit_size)
1335{
1336 const unsigned num_bits = dest_num_components * dest_bit_size;
1337
1338 /* Figure out the common bit size */
1339 unsigned common_bit_size = dest_bit_size;
1340 for (unsigned i = 0; i < num_srcs; i++)
1341 common_bit_size = MIN2(common_bit_size, srcs[i]->bit_size);
1342 if (first_bit > 0)
1343 common_bit_size = MIN2(common_bit_size, (1u << (ffs(first_bit) - 1)));
1344
1345 /* We don't want to have to deal with 1-bit values */
1346 assert(common_bit_size >= 8);
1347
1348 nir_def *common_comps[NIR_MAX_VEC_COMPONENTS * sizeof(uint64_t)];
1349 assert(num_bits / common_bit_size <= ARRAY_SIZE(common_comps));
1350
1351 /* First, unpack to the common bit size and select the components from the
1352 * source.
1353 */
1354 int src_idx = -1;
1355 unsigned src_start_bit = 0;
1356 unsigned src_end_bit = 0;
1357 for (unsigned i = 0; i < num_bits / common_bit_size; i++) {
1358 const unsigned bit = first_bit + (i * common_bit_size);
1359 while (bit >= src_end_bit) {
1360 src_idx++;
1361 assert(src_idx < (int)num_srcs);
1362 src_start_bit = src_end_bit;
1363 src_end_bit += srcs[src_idx]->bit_size *
1364 srcs[src_idx]->num_components;
1365 }
1366 assert(bit >= src_start_bit);
1367 assert(bit + common_bit_size <= src_end_bit);
1368 const unsigned rel_bit = bit - src_start_bit;
1369 const unsigned src_bit_size = srcs[src_idx]->bit_size;
1370
1371 nir_def *comp = nir_channel(b, srcs[src_idx],
1372 rel_bit / src_bit_size);
1373 if (srcs[src_idx]->bit_size > common_bit_size) {
1374 nir_def *unpacked = nir_unpack_bits(b, comp, common_bit_size);
1375 comp = nir_channel(b, unpacked, (rel_bit % src_bit_size) / common_bit_size);
1376 }
1377 common_comps[i] = comp;
1378 }
1379
1380 /* Now, re-pack the destination if we have to */
1381 if (dest_bit_size > common_bit_size) {
1382 unsigned common_per_dest = dest_bit_size / common_bit_size;
1383 nir_def *dest_comps[NIR_MAX_VEC_COMPONENTS];
1384 for (unsigned i = 0; i < dest_num_components; i++) {
1385 nir_def *unpacked = nir_vec(b, common_comps + i * common_per_dest,
1386 common_per_dest);
1387 dest_comps[i] = nir_pack_bits(b, unpacked, dest_bit_size);
1388 }
1389 return nir_vec(b, dest_comps, dest_num_components);
1390 } else {
1391 assert(dest_bit_size == common_bit_size);
1392 return nir_vec(b, common_comps, dest_num_components);
1393 }
1394}
1395
1396static inline nir_def *
1397nir_bitcast_vector(nir_builder *b, nir_def *src, unsigned dest_bit_size)
1398{
1399 assert((src->bit_size * src->num_components) % dest_bit_size == 0);
1400 const unsigned dest_num_components =
1401 (src->bit_size * src->num_components) / dest_bit_size;
1402 assert(dest_num_components <= NIR_MAX_VEC_COMPONENTS);
1403
1404 return nir_extract_bits(b, &src, 1, 0, dest_num_components, dest_bit_size);
1405}
1406
1407static inline nir_def *
1408nir_trim_vector(nir_builder *b, nir_def *src, unsigned num_components)
1409{
1410 assert(src->num_components >= num_components);
1411 if (src->num_components == num_components)
1412 return src;
1413
1414 return nir_channels(b, src, nir_component_mask(num_components));
1415}
1416
1417/**
1418 * Pad a value to N components with undefs of matching bit size.
1419 * If the value already contains >= num_components, it is returned without change.
1420 */
1421static inline nir_def *
1422nir_pad_vector(nir_builder *b, nir_def *src, unsigned num_components)
1423{
1424 assert(src->num_components <= num_components);
1425 if (src->num_components == num_components)
1426 return src;
1427
1428 nir_scalar components[NIR_MAX_VEC_COMPONENTS];
1429 nir_scalar undef = nir_get_scalar(nir_undef(b, 1, src->bit_size), 0);
1430 unsigned i = 0;
1431 for (; i < src->num_components; i++)
1432 components[i] = nir_get_scalar(src, i);
1433 for (; i < num_components; i++)
1434 components[i] = undef;
1435
1436 return nir_vec_scalars(b, components, num_components);
1437}
1438
1439/**
1440 * Pad a value to N components with copies of the given immediate of matching
1441 * bit size. If the value already contains >= num_components, it is returned
1442 * without change.
1443 */
1444static inline nir_def *
1445nir_pad_vector_imm_int(nir_builder *b, nir_def *src, uint64_t imm_val,
1446 unsigned num_components)
1447{
1448 assert(src->num_components <= num_components);
1449 if (src->num_components == num_components)
1450 return src;
1451
1452 nir_scalar components[NIR_MAX_VEC_COMPONENTS];
1453 nir_scalar imm = nir_get_scalar(nir_imm_intN_t(b, imm_val, src->bit_size), 0);
1454 unsigned i = 0;
1455 for (; i < src->num_components; i++)
1456 components[i] = nir_get_scalar(src, i);
1457 for (; i < num_components; i++)
1458 components[i] = imm;
1459
1460 return nir_vec_scalars(b, components, num_components);
1461}
1462
1463/**
1464 * Pad a value to 4 components with undefs of matching bit size.
1465 * If the value already contains >= 4 components, it is returned without change.
1466 */
1467static inline nir_def *
1468nir_pad_vec4(nir_builder *b, nir_def *src)
1469{
1470 return nir_pad_vector(b, src, 4);
1471}
1472
1473/**
1474 * Resizes a vector by either trimming off components or adding undef
1475 * components, as needed. Only use this helper if it's actually what you
1476 * need. Prefer nir_pad_vector() or nir_trim_vector() instead if you know a
1477 * priori which direction you're resizing.
1478 */
1479static inline nir_def *
1480nir_resize_vector(nir_builder *b, nir_def *src, unsigned num_components)
1481{
1482 if (src->num_components < num_components)
1483 return nir_pad_vector(b, src, num_components);
1484 else
1485 return nir_trim_vector(b, src, num_components);
1486}
1487
1488nir_def *
1489nir_ssa_for_alu_src(nir_builder *build, nir_alu_instr *instr, unsigned srcn);
1490
1491static inline unsigned
1492nir_get_ptr_bitsize(nir_shader *shader)
1493{
1494 if (shader->info.stage == MESA_SHADER_KERNEL)
1495 return shader->info.cs.ptr_size;
1496 return 32;
1497}
1498
1499static inline nir_deref_instr *
1500nir_build_deref_var(nir_builder *build, nir_variable *var)
1501{
1502 nir_deref_instr *deref =
1503 nir_deref_instr_create(build->shader, nir_deref_type_var);
1504
1505 deref->modes = (nir_variable_mode)var->data.mode;
1506 deref->type = var->type;
1507 deref->var = var;
1508
1509 nir_def_init(&deref->instr, &deref->def, 1,
1510 nir_get_ptr_bitsize(build->shader));
1511
1512 nir_builder_instr_insert(build, &deref->instr);
1513
1514 return deref;
1515}
1516
1517static inline nir_deref_instr *
1518nir_build_deref_array(nir_builder *build, nir_deref_instr *parent,
1519 nir_def *index)
1520{
1521 assert(glsl_type_is_array(parent->type) ||
1522 glsl_type_is_matrix(parent->type) ||
1523 glsl_type_is_vector(parent->type));
1524
1525 assert(index->bit_size == parent->def.bit_size);
1526
1527 nir_deref_instr *deref =
1528 nir_deref_instr_create(build->shader, nir_deref_type_array);
1529
1530 deref->modes = parent->modes;
1531 deref->type = glsl_get_array_element(parent->type);
1532 deref->parent = nir_src_for_ssa(&parent->def);
1533 deref->arr.index = nir_src_for_ssa(index);
1534
1535 nir_def_init(&deref->instr, &deref->def,
1536 parent->def.num_components, parent->def.bit_size);
1537
1538 nir_builder_instr_insert(build, &deref->instr);
1539
1540 return deref;
1541}
1542
1543static inline nir_deref_instr *
1544nir_build_deref_array_imm(nir_builder *build, nir_deref_instr *parent,
1545 int64_t index)
1546{
1547 nir_def *idx_ssa = nir_imm_intN_t(build, index,
1548 parent->def.bit_size);
1549
1550 return nir_build_deref_array(build, parent, idx_ssa);
1551}
1552
1553static inline nir_deref_instr *
1554nir_build_deref_ptr_as_array(nir_builder *build, nir_deref_instr *parent,
1555 nir_def *index)
1556{
1557 assert(parent->deref_type == nir_deref_type_array ||
1558 parent->deref_type == nir_deref_type_ptr_as_array ||
1559 parent->deref_type == nir_deref_type_cast);
1560
1561 assert(index->bit_size == parent->def.bit_size);
1562
1563 nir_deref_instr *deref =
1564 nir_deref_instr_create(build->shader, nir_deref_type_ptr_as_array);
1565
1566 deref->modes = parent->modes;
1567 deref->type = parent->type;
1568 deref->parent = nir_src_for_ssa(&parent->def);
1569 deref->arr.index = nir_src_for_ssa(index);
1570
1571 nir_def_init(&deref->instr, &deref->def,
1572 parent->def.num_components, parent->def.bit_size);
1573
1574 nir_builder_instr_insert(build, &deref->instr);
1575
1576 return deref;
1577}
1578
1579static inline nir_deref_instr *
1580nir_build_deref_array_wildcard(nir_builder *build, nir_deref_instr *parent)
1581{
1582 assert(glsl_type_is_array(parent->type) ||
1583 glsl_type_is_matrix(parent->type));
1584
1585 nir_deref_instr *deref =
1586 nir_deref_instr_create(build->shader, nir_deref_type_array_wildcard);
1587
1588 deref->modes = parent->modes;
1589 deref->type = glsl_get_array_element(parent->type);
1590 deref->parent = nir_src_for_ssa(&parent->def);
1591
1592 nir_def_init(&deref->instr, &deref->def,
1593 parent->def.num_components, parent->def.bit_size);
1594
1595 nir_builder_instr_insert(build, &deref->instr);
1596
1597 return deref;
1598}
1599
1600static inline nir_deref_instr *
1601nir_build_deref_struct(nir_builder *build, nir_deref_instr *parent,
1602 unsigned index)
1603{
1604 assert(glsl_type_is_struct_or_ifc(parent->type));
1605
1606 nir_deref_instr *deref =
1607 nir_deref_instr_create(build->shader, nir_deref_type_struct);
1608
1609 deref->modes = parent->modes;
1610 deref->type = glsl_get_struct_field(parent->type, index);
1611 deref->parent = nir_src_for_ssa(&parent->def);
1612 deref->strct.index = index;
1613
1614 nir_def_init(&deref->instr, &deref->def,
1615 parent->def.num_components, parent->def.bit_size);
1616
1617 nir_builder_instr_insert(build, &deref->instr);
1618
1619 return deref;
1620}
1621
1622static inline nir_deref_instr *
1623nir_build_deref_cast_with_alignment(nir_builder *build, nir_def *parent,
1624 nir_variable_mode modes,
1625 const struct glsl_type *type,
1626 unsigned ptr_stride,
1627 unsigned align_mul,
1628 unsigned align_offset)
1629{
1630 nir_deref_instr *deref =
1631 nir_deref_instr_create(build->shader, nir_deref_type_cast);
1632
1633 deref->modes = modes;
1634 deref->type = type;
1635 deref->parent = nir_src_for_ssa(parent);
1636 deref->cast.align_mul = align_mul;
1637 deref->cast.align_offset = align_offset;
1638 deref->cast.ptr_stride = ptr_stride;
1639
1640 nir_def_init(&deref->instr, &deref->def, parent->num_components,
1641 parent->bit_size);
1642
1643 nir_builder_instr_insert(build, &deref->instr);
1644
1645 return deref;
1646}
1647
1648static inline nir_deref_instr *
1649nir_build_deref_cast(nir_builder *build, nir_def *parent,
1650 nir_variable_mode modes, const struct glsl_type *type,
1651 unsigned ptr_stride)
1652{
1653 return nir_build_deref_cast_with_alignment(build, parent, modes, type,
1654 ptr_stride, 0, 0);
1655}
1656
1657static inline nir_deref_instr *
1658nir_alignment_deref_cast(nir_builder *build, nir_deref_instr *parent,
1659 uint32_t align_mul, uint32_t align_offset)
1660{
1661 nir_deref_instr *deref =
1662 nir_deref_instr_create(build->shader, nir_deref_type_cast);
1663
1664 deref->modes = parent->modes;
1665 deref->type = parent->type;
1666 deref->parent = nir_src_for_ssa(&parent->def);
1667 deref->cast.ptr_stride = nir_deref_instr_array_stride(deref);
1668 deref->cast.align_mul = align_mul;
1669 deref->cast.align_offset = align_offset;
1670
1671 nir_def_init(&deref->instr, &deref->def,
1672 parent->def.num_components, parent->def.bit_size);
1673
1674 nir_builder_instr_insert(build, &deref->instr);
1675
1676 return deref;
1677}
1678
1679/** Returns a deref that follows another but starting from the given parent
1680 *
1681 * The new deref will be the same type and take the same array or struct index
1682 * as the leader deref but it may have a different parent. This is very
1683 * useful for walking deref paths.
1684 */
1685static inline nir_deref_instr *
1686nir_build_deref_follower(nir_builder *b, nir_deref_instr *parent,
1687 nir_deref_instr *leader)
1688{
1689 /* If the derefs would have the same parent, don't make a new one */
1690 if (leader->parent.ssa == &parent->def)
1691 return leader;
1692
1693 UNUSED nir_deref_instr *leader_parent = nir_src_as_deref(leader->parent);
1694
1695 switch (leader->deref_type) {
1696 case nir_deref_type_var:
1697 unreachable("A var dereference cannot have a parent");
1698 break;
1699
1700 case nir_deref_type_array:
1701 case nir_deref_type_array_wildcard:
1702 assert(glsl_type_is_matrix(parent->type) ||
1703 glsl_type_is_array(parent->type) ||
1704 (leader->deref_type == nir_deref_type_array &&
1705 glsl_type_is_vector(parent->type)));
1706 assert(glsl_get_length(parent->type) ==
1707 glsl_get_length(leader_parent->type));
1708
1709 if (leader->deref_type == nir_deref_type_array) {
1710 nir_def *index = nir_i2iN(b, leader->arr.index.ssa,
1711 parent->def.bit_size);
1712 return nir_build_deref_array(b, parent, index);
1713 } else {
1714 return nir_build_deref_array_wildcard(b, parent);
1715 }
1716
1717 case nir_deref_type_struct:
1718 assert(glsl_type_is_struct_or_ifc(parent->type));
1719 assert(glsl_get_length(parent->type) ==
1720 glsl_get_length(leader_parent->type));
1721
1722 return nir_build_deref_struct(b, parent, leader->strct.index);
1723
1724 case nir_deref_type_cast:
1725 return nir_build_deref_cast_with_alignment(b, &parent->def,
1726 leader->modes,
1727 leader->type,
1728 leader->cast.ptr_stride,
1729 leader->cast.align_mul,
1730 leader->cast.align_offset);
1731
1732 case nir_deref_type_ptr_as_array: {
1733 assert(parent->deref_type == nir_deref_type_array ||
1734 parent->deref_type == nir_deref_type_ptr_as_array ||
1735 parent->deref_type == nir_deref_type_cast);
1736 nir_def *index = nir_i2iN(b, leader->arr.index.ssa,
1737 parent->def.bit_size);
1738 return nir_build_deref_ptr_as_array(b, parent, index);
1739 }
1740
1741 default:
1742 unreachable("Invalid deref instruction type");
1743 }
1744 return NULL;
1745}
1746
1747static inline nir_def *
1748nir_load_deref_with_access(nir_builder *build, nir_deref_instr *deref,
1749 enum gl_access_qualifier access)
1750{
1751 return nir_build_load_deref(build, glsl_get_vector_elements(deref->type),
1752 glsl_get_bit_size(deref->type), &deref->def,
1753 access);
1754}
1755
1756#undef nir_load_deref
1757static inline nir_def *
1758nir_load_deref(nir_builder *build, nir_deref_instr *deref)
1759{
1760 return nir_load_deref_with_access(build, deref, (enum gl_access_qualifier)0);
1761}
1762
1763static inline void
1764nir_store_deref_with_access(nir_builder *build, nir_deref_instr *deref,
1765 nir_def *value, unsigned writemask,
1766 enum gl_access_qualifier access)
1767{
1768 writemask &= (1u << value->num_components) - 1u;
1769 nir_build_store_deref(build, &deref->def, value, writemask, access);
1770}
1771
1772#undef nir_store_deref
1773static inline void
1774nir_store_deref(nir_builder *build, nir_deref_instr *deref,
1775 nir_def *value, unsigned writemask)
1776{
1777 nir_store_deref_with_access(build, deref, value, writemask,
1778 (enum gl_access_qualifier)0);
1779}
1780
1781static inline void
1782nir_build_write_masked_store(nir_builder *b, nir_deref_instr *vec_deref,
1783 nir_def *value, unsigned component)
1784{
1785 assert(value->num_components == 1);
1786 unsigned num_components = glsl_get_components(vec_deref->type);
1787 assert(num_components > 1 && num_components <= NIR_MAX_VEC_COMPONENTS);
1788
1789 nir_def *vec =
1790 nir_vector_insert_imm(b, nir_undef(b, num_components, value->bit_size),
1791 value, component);
1792 nir_store_deref(b, vec_deref, vec, (1u << component));
1793}
1794
1795static inline void
1796nir_build_write_masked_stores(nir_builder *b, nir_deref_instr *vec_deref,
1797 nir_def *value, nir_def *index,
1798 unsigned start, unsigned end)
1799{
1800 if (start == end - 1) {
1801 nir_build_write_masked_store(b, vec_deref, value, start);
1802 } else {
1803 unsigned mid = start + (end - start) / 2;
1804 nir_push_if(b, nir_ilt_imm(b, index, mid));
1805 nir_build_write_masked_stores(b, vec_deref, value, index, start, mid);
1806 nir_push_else(b, NULL);
1807 nir_build_write_masked_stores(b, vec_deref, value, index, mid, end);
1808 nir_pop_if(b, NULL);
1809 }
1810}
1811
1812static inline void
1813nir_copy_deref_with_access(nir_builder *build, nir_deref_instr *dest,
1814 nir_deref_instr *src,
1815 enum gl_access_qualifier dest_access,
1816 enum gl_access_qualifier src_access)
1817{
1818 nir_build_copy_deref(build, &dest->def, &src->def, dest_access, src_access);
1819}
1820
1821#undef nir_copy_deref
1822static inline void
1823nir_copy_deref(nir_builder *build, nir_deref_instr *dest, nir_deref_instr *src)
1824{
1825 nir_copy_deref_with_access(build, dest, src,
1826 (enum gl_access_qualifier)0,
1827 (enum gl_access_qualifier)0);
1828}
1829
1830static inline void
1831nir_memcpy_deref_with_access(nir_builder *build, nir_deref_instr *dest,
1832 nir_deref_instr *src, nir_def *size,
1833 enum gl_access_qualifier dest_access,
1834 enum gl_access_qualifier src_access)
1835{
1836 nir_build_memcpy_deref(build, &dest->def, &src->def,
1837 size, dest_access, src_access);
1838}
1839
1840#undef nir_memcpy_deref
1841static inline void
1842nir_memcpy_deref(nir_builder *build, nir_deref_instr *dest,
1843 nir_deref_instr *src, nir_def *size)
1844{
1845 nir_memcpy_deref_with_access(build, dest, src, size,
1846 (enum gl_access_qualifier)0,
1847 (enum gl_access_qualifier)0);
1848}
1849
1850static inline nir_def *
1851nir_load_var(nir_builder *build, nir_variable *var)
1852{
1853 return nir_load_deref(build, nir_build_deref_var(build, var));
1854}
1855
1856static inline void
1857nir_store_var(nir_builder *build, nir_variable *var, nir_def *value,
1858 unsigned writemask)
1859{
1860 nir_store_deref(build, nir_build_deref_var(build, var), value, writemask);
1861}
1862
1863static inline void
1864nir_copy_var(nir_builder *build, nir_variable *dest, nir_variable *src)
1865{
1866 nir_copy_deref(build, nir_build_deref_var(build, dest),
1867 nir_build_deref_var(build, src));
1868}
1869
1870static inline nir_def *
1871nir_load_array_var(nir_builder *build, nir_variable *var, nir_def *index)
1872{
1873 nir_deref_instr *deref =
1874 nir_build_deref_array(build, nir_build_deref_var(build, var), index);
1875 return nir_load_deref(build, deref);
1876}
1877
1878static inline nir_def *
1879nir_load_array_var_imm(nir_builder *build, nir_variable *var, int64_t index)
1880{
1881 nir_deref_instr *deref =
1882 nir_build_deref_array_imm(build, nir_build_deref_var(build, var), index);
1883 return nir_load_deref(build, deref);
1884}
1885
1886static inline void
1887nir_store_array_var(nir_builder *build, nir_variable *var, nir_def *index,
1888 nir_def *value, unsigned writemask)
1889{
1890 nir_deref_instr *deref =
1891 nir_build_deref_array(build, nir_build_deref_var(build, var), index);
1892 nir_store_deref(build, deref, value, writemask);
1893}
1894
1895static inline void
1896nir_store_array_var_imm(nir_builder *build, nir_variable *var, int64_t index,
1897 nir_def *value, unsigned writemask)
1898{
1899 nir_deref_instr *deref =
1900 nir_build_deref_array_imm(build, nir_build_deref_var(build, var), index);
1901 nir_store_deref(build, deref, value, writemask);
1902}
1903
1904#undef nir_load_global
1905static inline nir_def *
1906nir_load_global(nir_builder *build, nir_def *addr, unsigned align,
1907 unsigned num_components, unsigned bit_size)
1908{
1909 nir_intrinsic_instr *load =
1910 nir_intrinsic_instr_create(build->shader, nir_intrinsic_load_global);
1911 load->num_components = (uint8_t)num_components;
1912 load->src[0] = nir_src_for_ssa(addr);
1913 nir_intrinsic_set_align(load, align, 0);
1914 nir_def_init(&load->instr, &load->def, num_components, bit_size);
1915 nir_builder_instr_insert(build, &load->instr);
1916 return &load->def;
1917}
1918
1919#undef nir_store_global
1920static inline void
1921nir_store_global(nir_builder *build, nir_def *addr, unsigned align,
1922 nir_def *value, nir_component_mask_t write_mask)
1923{
1924 nir_intrinsic_instr *store =
1925 nir_intrinsic_instr_create(build->shader, nir_intrinsic_store_global);
1926 store->num_components = value->num_components;
1927 store->src[0] = nir_src_for_ssa(value);
1928 store->src[1] = nir_src_for_ssa(addr);
1929 nir_intrinsic_set_write_mask(store,
1930 write_mask & BITFIELD_MASK(value->num_components));
1931 nir_intrinsic_set_align(store, align, 0);
1932 nir_builder_instr_insert(build, &store->instr);
1933}
1934
1935#undef nir_load_global_constant
1936static inline nir_def *
1937nir_load_global_constant(nir_builder *build, nir_def *addr, unsigned align,
1938 unsigned num_components, unsigned bit_size)
1939{
1940 nir_intrinsic_instr *load =
1941 nir_intrinsic_instr_create(build->shader, nir_intrinsic_load_global_constant);
1942 load->num_components = (uint8_t)num_components;
1943 load->src[0] = nir_src_for_ssa(addr);
1944 nir_intrinsic_set_align(load, align, 0);
1945 nir_def_init(&load->instr, &load->def, num_components, bit_size);
1946 nir_builder_instr_insert(build, &load->instr);
1947 return &load->def;
1948}
1949
1950#undef nir_load_param
1951static inline nir_def *
1952nir_load_param(nir_builder *build, uint32_t param_idx)
1953{
1954 assert(param_idx < build->impl->function->num_params);
1955 nir_parameter *param = &build->impl->function->params[param_idx];
1956 return nir_build_load_param(build, param->num_components, param->bit_size, param_idx);
1957}
1958
1959#undef nir_decl_reg
1960static inline nir_def *
1961nir_decl_reg(nir_builder *b, unsigned num_components, unsigned bit_size,
1962 unsigned num_array_elems)
1963{
1964 nir_intrinsic_instr *decl =
1965 nir_intrinsic_instr_create(b->shader, nir_intrinsic_decl_reg);
1966 nir_intrinsic_set_num_components(decl, num_components);
1967 nir_intrinsic_set_bit_size(decl, bit_size);
1968 nir_intrinsic_set_num_array_elems(decl, num_array_elems);
1969 nir_intrinsic_set_divergent(decl, true);
1970 nir_def_init(&decl->instr, &decl->def, 1, 32);
1971
1972 nir_builder_instr_insert_at_top(b, &decl->instr);
1973
1974 return &decl->def;
1975}
1976
1977#undef nir_load_reg
1978static inline nir_def *
1979nir_load_reg(nir_builder *b, nir_def *reg)
1980{
1981 nir_intrinsic_instr *decl = nir_reg_get_decl(reg);
1982 unsigned num_components = nir_intrinsic_num_components(decl);
1983 unsigned bit_size = nir_intrinsic_bit_size(decl);
1984
1985 nir_def *res = nir_build_load_reg(b, num_components, bit_size, reg);
1986 res->divergent = nir_intrinsic_divergent(decl);
1987
1988 return res;
1989}
1990
1991#undef nir_store_reg
1992static inline void
1993nir_store_reg(nir_builder *b, nir_def *value, nir_def *reg)
1994{
1995 ASSERTED nir_intrinsic_instr *decl = nir_reg_get_decl(reg);
1996 ASSERTED unsigned num_components = nir_intrinsic_num_components(decl);
1997 ASSERTED unsigned bit_size = nir_intrinsic_bit_size(decl);
1998
1999 assert(value->num_components == num_components);
2000 assert(value->bit_size == bit_size);
2001
2002 nir_build_store_reg(b, value, reg);
2003}
2004
2005static inline nir_tex_src
2006nir_tex_src_for_ssa(nir_tex_src_type src_type, nir_def *def)
2007{
2008 nir_tex_src src;
2009 src.src = nir_src_for_ssa(def);
2010 src.src_type = src_type;
2011 return src;
2012}
2013
2014#undef nir_ddx
2015#undef nir_ddx_fine
2016#undef nir_ddx_coarse
2017#undef nir_ddy
2018#undef nir_ddy_fine
2019#undef nir_ddy_coarse
2020
2021static inline nir_def *
2022nir_build_deriv(nir_builder *b, nir_def *x, nir_intrinsic_op intrin)
2023{
2024 if (b->shader->options->scalarize_ddx && x->num_components > 1) {
2025 nir_def *res[NIR_MAX_VEC_COMPONENTS] = { NULL };
2026
2027 for (unsigned i = 0; i < x->num_components; ++i) {
2028 res[i] = _nir_build_ddx(b, x->bit_size, nir_channel(b, x, i));
2029 nir_instr_as_intrinsic(res[i]->parent_instr)->intrinsic = intrin;
2030 }
2031
2032 return nir_vec(b, res, x->num_components);
2033 } else {
2034 nir_def *res = _nir_build_ddx(b, x->bit_size, x);
2035 nir_instr_as_intrinsic(res->parent_instr)->intrinsic = intrin;
2036 return res;
2037 }
2038}
2039
2040#define DEF_DERIV(op) \
2041 static inline nir_def * \
2042 nir_##op(nir_builder *build, nir_def *src0) \
2043 { \
2044 return nir_build_deriv(build, src0, nir_intrinsic_##op); \
2045 }
2046
2047DEF_DERIV(ddx)
2048DEF_DERIV(ddx_fine)
2049DEF_DERIV(ddx_coarse)
2050DEF_DERIV(ddy)
2051DEF_DERIV(ddy_fine)
2052DEF_DERIV(ddy_coarse)
2053
2054/*
2055 * Find a texture source, remove it, and return its nir_def. If the texture
2056 * source does not exist, return NULL. This is useful for texture lowering pass
2057 * that consume their input sources and produce a new lowered source.
2058 */
2059static inline nir_def *
2060nir_steal_tex_src(nir_tex_instr *tex, nir_tex_src_type type_)
2061{
2062 int idx = nir_tex_instr_src_index(tex, type_);
2063 if (idx < 0)
2064 return NULL;
2065
2066 nir_def *ssa = tex->src[idx].src.ssa;
2067 nir_tex_instr_remove_src(tex, idx);
2068 return ssa;
2069}
2070
2071static inline nir_def *
2072nir_tex_deref(nir_builder *b, nir_deref_instr *t, nir_deref_instr *s,
2073 nir_def *coord)
2074{
2075 nir_tex_src srcs[] = { nir_tex_src_for_ssa(nir_tex_src_coord, coord) };
2076
2077 return nir_build_tex_deref_instr(b, nir_texop_tex, t, s,
2078 ARRAY_SIZE(srcs), srcs);
2079}
2080
2081static inline nir_def *
2082nir_txl_deref(nir_builder *b, nir_deref_instr *t, nir_deref_instr *s,
2083 nir_def *coord, nir_def *lod)
2084{
2085 nir_tex_src srcs[] = {
2086 nir_tex_src_for_ssa(nir_tex_src_coord, coord),
2087 nir_tex_src_for_ssa(nir_tex_src_lod, lod),
2088 };
2089
2090 return nir_build_tex_deref_instr(b, nir_texop_txl, t, s,
2091 ARRAY_SIZE(srcs), srcs);
2092}
2093
2094static inline nir_def *
2095nir_txl_zero_deref(nir_builder *b, nir_deref_instr *t, nir_deref_instr *s,
2096 nir_def *coord)
2097{
2098 return nir_txl_deref(b, t, s, coord, nir_imm_float(b, 0));
2099}
2100
2101static inline bool
2102nir_tex_type_has_lod(const struct glsl_type *tex_type)
2103{
2104 switch (glsl_get_sampler_dim(tex_type)) {
2105 case GLSL_SAMPLER_DIM_1D:
2106 case GLSL_SAMPLER_DIM_2D:
2107 case GLSL_SAMPLER_DIM_3D:
2108 case GLSL_SAMPLER_DIM_CUBE:
2109 return true;
2110 default:
2111 return false;
2112 }
2113}
2114
2115static inline nir_def *
2116nir_txf_deref(nir_builder *b, nir_deref_instr *t,
2117 nir_def *coord, nir_def *lod)
2118{
2119 nir_tex_src srcs[2];
2120 unsigned num_srcs = 0;
2121
2122 srcs[num_srcs++] = nir_tex_src_for_ssa(nir_tex_src_coord, coord);
2123
2124 if (lod == NULL && nir_tex_type_has_lod(t->type))
2125 lod = nir_imm_int(b, 0);
2126
2127 if (lod != NULL)
2128 srcs[num_srcs++] = nir_tex_src_for_ssa(nir_tex_src_lod, lod);
2129
2130 return nir_build_tex_deref_instr(b, nir_texop_txf, t, NULL,
2131 num_srcs, srcs);
2132}
2133
2134static inline nir_def *
2135nir_txf_ms_deref(nir_builder *b, nir_deref_instr *t,
2136 nir_def *coord, nir_def *ms_index)
2137{
2138 nir_tex_src srcs[] = {
2139 nir_tex_src_for_ssa(nir_tex_src_coord, coord),
2140 nir_tex_src_for_ssa(nir_tex_src_ms_index, ms_index),
2141 };
2142
2143 return nir_build_tex_deref_instr(b, nir_texop_txf_ms, t, NULL,
2144 ARRAY_SIZE(srcs), srcs);
2145}
2146
2147static inline nir_def *
2148nir_txs_deref(nir_builder *b, nir_deref_instr *t, nir_def *lod)
2149{
2150 nir_tex_src srcs[1];
2151 unsigned num_srcs = 0;
2152
2153 if (lod == NULL && nir_tex_type_has_lod(t->type))
2154 lod = nir_imm_int(b, 0);
2155
2156 if (lod != NULL)
2157 srcs[num_srcs++] = nir_tex_src_for_ssa(nir_tex_src_lod, lod);
2158
2159 return nir_build_tex_deref_instr(b, nir_texop_txs, t, NULL,
2160 num_srcs, srcs);
2161}
2162
2163static inline nir_def *
2164nir_samples_identical_deref(nir_builder *b, nir_deref_instr *t,
2165 nir_def *coord)
2166{
2167 nir_tex_src srcs[] = { nir_tex_src_for_ssa(nir_tex_src_coord, coord) };
2168
2169 return nir_build_tex_deref_instr(b, nir_texop_samples_identical, t, NULL,
2170 ARRAY_SIZE(srcs), srcs);
2171}
2172
2173/* calculate a `(1 << value) - 1` in ssa without overflows */
2174static inline nir_def *
2175nir_mask(nir_builder *b, nir_def *bits, unsigned dst_bit_size)
2176{
2177 return nir_ushr(b, nir_imm_intN_t(b, -1, dst_bit_size),
2178 nir_isub_imm(b, dst_bit_size, nir_u2u32(b, bits)));
2179}
2180
2181static inline nir_def *
2182nir_load_barycentric(nir_builder *build, nir_intrinsic_op op,
2183 unsigned interp_mode)
2184{
2185 unsigned num_components = op == nir_intrinsic_load_barycentric_model ? 3 : 2;
2186 nir_intrinsic_instr *bary = nir_intrinsic_instr_create(build->shader, op);
2187 nir_def_init(&bary->instr, &bary->def, num_components, 32);
2188 nir_intrinsic_set_interp_mode(bary, interp_mode);
2189 nir_builder_instr_insert(build, &bary->instr);
2190 return &bary->def;
2191}
2192
2193static inline void
2194nir_jump(nir_builder *build, nir_jump_type jump_type)
2195{
2196 assert(jump_type != nir_jump_goto && jump_type != nir_jump_goto_if);
2197 nir_jump_instr *jump = nir_jump_instr_create(build->shader, jump_type);
2198 nir_builder_instr_insert(build, &jump->instr);
2199}
2200
2201static inline void
2202nir_goto(nir_builder *build, struct nir_block *target)
2203{
2204 assert(!build->impl->structured);
2205 nir_jump_instr *jump = nir_jump_instr_create(build->shader, nir_jump_goto);
2206 jump->target = target;
2207 nir_builder_instr_insert(build, &jump->instr);
2208}
2209
2210static inline void
2211nir_goto_if(nir_builder *build, struct nir_block *target, nir_def *cond,
2212 struct nir_block *else_target)
2213{
2214 assert(!build->impl->structured);
2215 nir_jump_instr *jump = nir_jump_instr_create(build->shader, nir_jump_goto_if);
2216 jump->condition = nir_src_for_ssa(cond);
2217 jump->target = target;
2218 jump->else_target = else_target;
2219 nir_builder_instr_insert(build, &jump->instr);
2220}
2221
2222static inline void
2223nir_break_if(nir_builder *build, nir_def *cond)
2224{
2225 nir_if *nif = nir_push_if(build, cond);
2226 {
2227 nir_jump(build, nir_jump_break);
2228 }
2229 nir_pop_if(build, nif);
2230}
2231
2232static inline void
2233nir_build_call(nir_builder *build, nir_function *func, size_t count,
2234 nir_def **args)
2235{
2236 assert(count == func->num_params && "parameter count must match");
2237 nir_call_instr *call = nir_call_instr_create(build->shader, func);
2238
2239 for (unsigned i = 0; i < count; ++i) {
2240 call->params[i] = nir_src_for_ssa(args[i]);
2241 }
2242
2243 nir_builder_instr_insert(build, &call->instr);
2244}
2245
2246static inline void
2247nir_build_indirect_call(nir_builder *build, nir_function *func, nir_def *callee,
2248 size_t count, nir_def **args)
2249{
2250 assert(count == func->num_params && "parameter count must match");
2251 assert(!func->impl && "cannot call directly defined functions indirectly");
2252 nir_call_instr *call = nir_call_instr_create(build->shader, func);
2253
2254 for (unsigned i = 0; i < func->num_params; ++i) {
2255 call->params[i] = nir_src_for_ssa(args[i]);
2256 }
2257 call->indirect_callee = nir_src_for_ssa(callee);
2258
2259 nir_builder_instr_insert(build, &call->instr);
2260}
2261
2262static inline void
2263nir_discard(nir_builder *build)
2264{
2265 if (build->shader->options->discard_is_demote)
2266 nir_demote(build);
2267 else
2268 nir_terminate(build);
2269}
2270
2271static inline void
2272nir_discard_if(nir_builder *build, nir_def *src)
2273{
2274 if (build->shader->options->discard_is_demote)
2275 nir_demote_if(build, src);
2276 else
2277 nir_terminate_if(build, src);
2278}
2279
2280nir_def *
2281nir_build_string(nir_builder *build, const char *value);
2282
2283/*
2284 * Call a given nir_function * with a variadic number of nir_def * arguments.
2285 *
2286 * Defined with __VA_ARGS__ instead of va_list so we can assert the correct
2287 * number of parameters are passed in.
2288 */
2289#define nir_call(build, func, ...) \
2290 do { \
2291 nir_def *args[] = { __VA_ARGS__ }; \
2292 nir_build_call(build, func, ARRAY_SIZE(args), args); \
2293 } while (0)
2294
2295#define nir_call_indirect(build, func, callee, ...) \
2296 do { \
2297 nir_def *_args[] = { __VA_ARGS__ }; \
2298 nir_build_indirect_call(build, func, callee, ARRAY_SIZE(_args), _args); \
2299 } while (0)
2300
2301nir_def *
2302nir_compare_func(nir_builder *b, enum compare_func func,
2303 nir_def *src0, nir_def *src1);
2304
2305static inline void
2306nir_scoped_memory_barrier(nir_builder *b,
2307 mesa_scope scope,
2308 nir_memory_semantics semantics,
2309 nir_variable_mode modes)
2310{
2311 nir_barrier(b, SCOPE_NONE, scope, semantics, modes);
2312}
2313
2314nir_def *
2315nir_gen_rect_vertices(nir_builder *b, nir_def *z, nir_def *w);
2316
2317/* Emits a printf in the same way nir_lower_printf(). Each of the variadic
2318 * argument is a pointer to a nir_def value.
2319 */
2320void nir_printf_fmt(nir_builder *b,
2321 bool use_printf_base_identifier,
2322 unsigned ptr_bit_size,
2323 const char *fmt, ...);
2324
2325#ifdef __cplusplus
2326} /* extern "C" */
2327#endif
2328
2329#endif /* NIR_BUILDER_H */