1/*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 *    Jason Ekstrand (jason@jlekstrand.net)
25 *
26 */
27
28#include "nir/nir.h"
29#include "nir/nir_builder.h"
30#include "nir/nir_array.h"
31#include "nir_spirv.h"
32#include "spirv.h"
33
34struct vtn_builder;
35struct vtn_decoration;
36
37enum vtn_value_type {
38   vtn_value_type_invalid = 0,
39   vtn_value_type_undef,
40   vtn_value_type_string,
41   vtn_value_type_decoration_group,
42   vtn_value_type_type,
43   vtn_value_type_constant,
44   vtn_value_type_access_chain,
45   vtn_value_type_function,
46   vtn_value_type_block,
47   vtn_value_type_ssa,
48   vtn_value_type_extension,
49   vtn_value_type_image_pointer,
50   vtn_value_type_sampled_image,
51};
52
53enum vtn_branch_type {
54   vtn_branch_type_none,
55   vtn_branch_type_switch_break,
56   vtn_branch_type_switch_fallthrough,
57   vtn_branch_type_loop_break,
58   vtn_branch_type_loop_continue,
59   vtn_branch_type_discard,
60   vtn_branch_type_return,
61};
62
63enum vtn_cf_node_type {
64   vtn_cf_node_type_block,
65   vtn_cf_node_type_if,
66   vtn_cf_node_type_loop,
67   vtn_cf_node_type_switch,
68};
69
70struct vtn_cf_node {
71   struct list_head link;
72   enum vtn_cf_node_type type;
73};
74
75struct vtn_loop {
76   struct vtn_cf_node node;
77
78   /* The main body of the loop */
79   struct list_head body;
80
81   /* The "continue" part of the loop.  This gets executed after the body
82    * and is where you go when you hit a continue.
83    */
84   struct list_head cont_body;
85
86   SpvLoopControlMask control;
87};
88
89struct vtn_if {
90   struct vtn_cf_node node;
91
92   uint32_t condition;
93
94   enum vtn_branch_type then_type;
95   struct list_head then_body;
96
97   enum vtn_branch_type else_type;
98   struct list_head else_body;
99
100   SpvSelectionControlMask control;
101};
102
103struct vtn_case {
104   struct list_head link;
105
106   struct list_head body;
107
108   /* The block that starts this case */
109   struct vtn_block *start_block;
110
111   /* The fallthrough case, if any */
112   struct vtn_case *fallthrough;
113
114   /* The uint32_t values that map to this case */
115   nir_array values;
116
117   /* True if this is the default case */
118   bool is_default;
119
120   /* Initialized to false; used when sorting the list of cases */
121   bool visited;
122};
123
124struct vtn_switch {
125   struct vtn_cf_node node;
126
127   uint32_t selector;
128
129   struct list_head cases;
130};
131
132struct vtn_block {
133   struct vtn_cf_node node;
134
135   /** A pointer to the label instruction */
136   const uint32_t *label;
137
138   /** A pointer to the merge instruction (or NULL if non exists) */
139   const uint32_t *merge;
140
141   /** A pointer to the branch instruction that ends this block */
142   const uint32_t *branch;
143
144   enum vtn_branch_type branch_type;
145
146   /** Points to the loop that this block starts (if it starts a loop) */
147   struct vtn_loop *loop;
148
149   /** Points to the switch case started by this block (if any) */
150   struct vtn_case *switch_case;
151
152   /** Every block ends in a nop intrinsic so that we can find it again */
153   nir_intrinsic_instr *end_nop;
154};
155
156struct vtn_function {
157   struct exec_node node;
158
159   nir_function_impl *impl;
160   struct vtn_block *start_block;
161
162   struct list_head body;
163
164   const uint32_t *end;
165
166   SpvFunctionControlMask control;
167};
168
169typedef bool (*vtn_instruction_handler)(struct vtn_builder *, uint32_t,
170                                        const uint32_t *, unsigned);
171
172void vtn_build_cfg(struct vtn_builder *b, const uint32_t *words,
173                   const uint32_t *end);
174void vtn_function_emit(struct vtn_builder *b, struct vtn_function *func,
175                       vtn_instruction_handler instruction_handler);
176
177const uint32_t *
178vtn_foreach_instruction(struct vtn_builder *b, const uint32_t *start,
179                        const uint32_t *end, vtn_instruction_handler handler);
180
181struct vtn_ssa_value {
182   union {
183      nir_ssa_def *def;
184      struct vtn_ssa_value **elems;
185   };
186
187   /* For matrices, if this is non-NULL, then this value is actually the
188    * transpose of some other value.  The value that `transposed` points to
189    * always dominates this value.
190    */
191   struct vtn_ssa_value *transposed;
192
193   const struct glsl_type *type;
194};
195
196struct vtn_type {
197   const struct glsl_type *type;
198
199   /* The value that declares this type.  Used for finding decorations */
200   struct vtn_value *val;
201
202   /* for matrices, whether the matrix is stored row-major */
203   bool row_major;
204
205   /* for structs, the offset of each member */
206   unsigned *offsets;
207
208   /* for structs, whether it was decorated as a "non-SSBO-like" block */
209   bool block;
210
211   /* for structs, whether it was decorated as an "SSBO-like" block */
212   bool buffer_block;
213
214   /* for structs with block == true, whether this is a builtin block (i.e. a
215    * block that contains only builtins).
216    */
217   bool builtin_block;
218
219   /* Image format for image_load_store type images */
220   unsigned image_format;
221
222   /* Access qualifier for storage images */
223   SpvAccessQualifier access_qualifier;
224
225   /* for arrays and matrices, the array stride */
226   unsigned stride;
227
228   /* for arrays, the vtn_type for the elements of the array */
229   struct vtn_type *array_element;
230
231   /* for structures, the vtn_type for each member */
232   struct vtn_type **members;
233
234   /* Whether this type, or a parent type, has been decorated as a builtin */
235   bool is_builtin;
236
237   SpvBuiltIn builtin;
238};
239
240struct vtn_variable;
241
242enum vtn_access_mode {
243   vtn_access_mode_id,
244   vtn_access_mode_literal,
245};
246
247struct vtn_access_link {
248   enum vtn_access_mode mode;
249   uint32_t id;
250};
251
252struct vtn_access_chain {
253   struct vtn_variable *var;
254
255   uint32_t length;
256
257   /* Struct elements and array offsets */
258   struct vtn_access_link link[0];
259};
260
261enum vtn_variable_mode {
262   vtn_variable_mode_local,
263   vtn_variable_mode_global,
264   vtn_variable_mode_param,
265   vtn_variable_mode_ubo,
266   vtn_variable_mode_ssbo,
267   vtn_variable_mode_push_constant,
268   vtn_variable_mode_image,
269   vtn_variable_mode_sampler,
270   vtn_variable_mode_workgroup,
271   vtn_variable_mode_input,
272   vtn_variable_mode_output,
273};
274
275struct vtn_variable {
276   enum vtn_variable_mode mode;
277
278   struct vtn_type *type;
279
280   unsigned descriptor_set;
281   unsigned binding;
282   unsigned input_attachment_index;
283   bool patch;
284
285   nir_variable *var;
286   nir_variable **members;
287
288   struct vtn_access_chain chain;
289};
290
291struct vtn_image_pointer {
292   struct vtn_access_chain *image;
293   nir_ssa_def *coord;
294   nir_ssa_def *sample;
295};
296
297struct vtn_sampled_image {
298   struct vtn_access_chain *image; /* Image or array of images */
299   struct vtn_access_chain *sampler; /* Sampler */
300};
301
302struct vtn_value {
303   enum vtn_value_type value_type;
304   const char *name;
305   struct vtn_decoration *decoration;
306   union {
307      void *ptr;
308      char *str;
309      struct vtn_type *type;
310      struct {
311         nir_constant *constant;
312         const struct glsl_type *const_type;
313      };
314      struct vtn_access_chain *access_chain;
315      struct vtn_image_pointer *image;
316      struct vtn_sampled_image *sampled_image;
317      struct vtn_function *func;
318      struct vtn_block *block;
319      struct vtn_ssa_value *ssa;
320      vtn_instruction_handler ext_handler;
321   };
322};
323
324#define VTN_DEC_DECORATION -1
325#define VTN_DEC_EXECUTION_MODE -2
326#define VTN_DEC_STRUCT_MEMBER0 0
327
328struct vtn_decoration {
329   struct vtn_decoration *next;
330
331   /* Specifies how to apply this decoration.  Negative values represent a
332    * decoration or execution mode. (See the VTN_DEC_ #defines above.)
333    * Non-negative values specify that it applies to a structure member.
334    */
335   int scope;
336
337   const uint32_t *literals;
338   struct vtn_value *group;
339
340   union {
341      SpvDecoration decoration;
342      SpvExecutionMode exec_mode;
343   };
344};
345
346struct vtn_builder {
347   nir_builder nb;
348
349   nir_shader *shader;
350   nir_function_impl *impl;
351   const struct nir_spirv_supported_extensions *ext;
352   struct vtn_block *block;
353
354   /* Current file, line, and column.  Useful for debugging.  Set
355    * automatically by vtn_foreach_instruction.
356    */
357   char *file;
358   int line, col;
359
360   /*
361    * In SPIR-V, constants are global, whereas in NIR, the load_const
362    * instruction we use is per-function. So while we parse each function, we
363    * keep a hash table of constants we've resolved to nir_ssa_value's so
364    * far, and we lazily resolve them when we see them used in a function.
365    */
366   struct hash_table *const_table;
367
368   /*
369    * Map from phi instructions (pointer to the start of the instruction)
370    * to the variable corresponding to it.
371    */
372   struct hash_table *phi_table;
373
374   unsigned num_specializations;
375   struct nir_spirv_specialization *specializations;
376
377   unsigned value_id_bound;
378   struct vtn_value *values;
379
380   gl_shader_stage entry_point_stage;
381   const char *entry_point_name;
382   struct vtn_value *entry_point;
383   bool origin_upper_left;
384   bool pixel_center_integer;
385
386   struct vtn_function *func;
387   struct exec_list functions;
388
389   /* Current function parameter index */
390   unsigned func_param_idx;
391
392   bool has_loop_continue;
393};
394
395static inline struct vtn_value *
396vtn_push_value(struct vtn_builder *b, uint32_t value_id,
397               enum vtn_value_type value_type)
398{
399   assert(value_id < b->value_id_bound);
400   assert(b->values[value_id].value_type == vtn_value_type_invalid);
401
402   b->values[value_id].value_type = value_type;
403
404   return &b->values[value_id];
405}
406
407static inline struct vtn_value *
408vtn_untyped_value(struct vtn_builder *b, uint32_t value_id)
409{
410   assert(value_id < b->value_id_bound);
411   return &b->values[value_id];
412}
413
414static inline struct vtn_value *
415vtn_value(struct vtn_builder *b, uint32_t value_id,
416          enum vtn_value_type value_type)
417{
418   struct vtn_value *val = vtn_untyped_value(b, value_id);
419   assert(val->value_type == value_type);
420   return val;
421}
422
423void _vtn_warn(const char *file, int line, const char *msg, ...);
424#define vtn_warn(...) _vtn_warn(__FILE__, __LINE__, __VA_ARGS__)
425
426struct vtn_ssa_value *vtn_ssa_value(struct vtn_builder *b, uint32_t value_id);
427
428struct vtn_ssa_value *vtn_create_ssa_value(struct vtn_builder *b,
429                                           const struct glsl_type *type);
430
431struct vtn_ssa_value *vtn_ssa_transpose(struct vtn_builder *b,
432                                        struct vtn_ssa_value *src);
433
434nir_ssa_def *vtn_vector_extract(struct vtn_builder *b, nir_ssa_def *src,
435                                unsigned index);
436nir_ssa_def *vtn_vector_extract_dynamic(struct vtn_builder *b, nir_ssa_def *src,
437                                        nir_ssa_def *index);
438nir_ssa_def *vtn_vector_insert(struct vtn_builder *b, nir_ssa_def *src,
439                               nir_ssa_def *insert, unsigned index);
440nir_ssa_def *vtn_vector_insert_dynamic(struct vtn_builder *b, nir_ssa_def *src,
441                                       nir_ssa_def *insert, nir_ssa_def *index);
442
443nir_deref_var *vtn_nir_deref(struct vtn_builder *b, uint32_t id);
444
445nir_deref_var *vtn_access_chain_to_deref(struct vtn_builder *b,
446                                         struct vtn_access_chain *chain);
447nir_ssa_def *
448vtn_access_chain_to_offset(struct vtn_builder *b,
449                           struct vtn_access_chain *chain,
450                           nir_ssa_def **index_out, struct vtn_type **type_out,
451                           unsigned *end_idx_out, bool stop_at_matrix);
452
453struct vtn_ssa_value *vtn_local_load(struct vtn_builder *b, nir_deref_var *src);
454
455void vtn_local_store(struct vtn_builder *b, struct vtn_ssa_value *src,
456                     nir_deref_var *dest);
457
458struct vtn_ssa_value *
459vtn_variable_load(struct vtn_builder *b, struct vtn_access_chain *src);
460
461void vtn_variable_store(struct vtn_builder *b, struct vtn_ssa_value *src,
462                        struct vtn_access_chain *dest);
463
464void vtn_handle_variables(struct vtn_builder *b, SpvOp opcode,
465                          const uint32_t *w, unsigned count);
466
467
468typedef void (*vtn_decoration_foreach_cb)(struct vtn_builder *,
469                                          struct vtn_value *,
470                                          int member,
471                                          const struct vtn_decoration *,
472                                          void *);
473
474void vtn_foreach_decoration(struct vtn_builder *b, struct vtn_value *value,
475                            vtn_decoration_foreach_cb cb, void *data);
476
477typedef void (*vtn_execution_mode_foreach_cb)(struct vtn_builder *,
478                                              struct vtn_value *,
479                                              const struct vtn_decoration *,
480                                              void *);
481
482void vtn_foreach_execution_mode(struct vtn_builder *b, struct vtn_value *value,
483                                vtn_execution_mode_foreach_cb cb, void *data);
484
485nir_op vtn_nir_alu_op_for_spirv_opcode(SpvOp opcode, bool *swap,
486                                       nir_alu_type src, nir_alu_type dst);
487
488void vtn_handle_alu(struct vtn_builder *b, SpvOp opcode,
489                    const uint32_t *w, unsigned count);
490
491bool vtn_handle_glsl450_instruction(struct vtn_builder *b, uint32_t ext_opcode,
492                                    const uint32_t *words, unsigned count);
493
494static inline uint64_t
495vtn_u64_literal(const uint32_t *w)
496{
497   return (uint64_t)w[1] << 32 | w[0];
498}
499