1/*
2 * Copyright 2011 Christoph Bumiller
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17 * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
18 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
19 * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20 * SOFTWARE.
21 */
22
23#ifndef __NV50_IR_H__
24#define __NV50_IR_H__
25
26#include <stdio.h>
27#include <stdlib.h>
28#include <stdint.h>
29#include <deque>
30#include <list>
31#include <vector>
32
33#include "nv50_ir_util.h"
34#include "nv50_ir_graph.h"
35
36#include "nv50_ir_driver.h"
37
38namespace nv50_ir {
39
40enum operation
41{
42   OP_NOP = 0,
43   OP_PHI,
44   OP_UNION, // unify a new definition and several source values
45   OP_SPLIT, // $r0d -> { $r0, $r1 } ($r0d and $r0/$r1 will be coalesced)
46   OP_MERGE, // opposite of split, e.g. combine 2 32 bit into a 64 bit value
47   OP_CONSTRAINT, // copy values into consecutive registers
48   OP_MOV, // simple copy, no modifiers allowed
49   OP_LOAD,
50   OP_STORE,
51   OP_ADD,
52   OP_SUB,
53   OP_MUL,
54   OP_DIV,
55   OP_MOD,
56   OP_MAD,
57   OP_FMA,
58   OP_SAD, // abs(src0 - src1) + src2
59   OP_ABS,
60   OP_NEG,
61   OP_NOT,
62   OP_AND,
63   OP_OR,
64   OP_XOR,
65   OP_SHL,
66   OP_SHR,
67   OP_MAX,
68   OP_MIN,
69   OP_SAT,  // CLAMP(f32, 0.0, 1.0)
70   OP_CEIL,
71   OP_FLOOR,
72   OP_TRUNC,
73   OP_CVT,
74   OP_SET_AND, // dst = (src0 CMP src1) & src2
75   OP_SET_OR,
76   OP_SET_XOR,
77   OP_SET,
78   OP_SELP, // dst = src2 ? src0 : src1
79   OP_SLCT, // dst = (src2 CMP 0) ? src0 : src1
80   OP_RCP,
81   OP_RSQ,
82   OP_LG2,
83   OP_SIN,
84   OP_COS,
85   OP_EX2,
86   OP_EXP, // exponential (base M_E)
87   OP_LOG, // natural logarithm
88   OP_PRESIN,
89   OP_PREEX2,
90   OP_SQRT,
91   OP_POW,
92   OP_BRA,
93   OP_CALL,
94   OP_RET,
95   OP_CONT,
96   OP_BREAK,
97   OP_PRERET,
98   OP_PRECONT,
99   OP_PREBREAK,
100   OP_BRKPT,     // breakpoint (not related to loops)
101   OP_JOINAT,    // push control flow convergence point
102   OP_JOIN,      // converge
103   OP_DISCARD,
104   OP_EXIT,
105   OP_MEMBAR,
106   OP_VFETCH, // indirection 0 in attribute space, indirection 1 is vertex base
107   OP_PFETCH, // fetch base address of vertex src0 (immediate) [+ src1]
108   OP_EXPORT,
109   OP_LINTERP,
110   OP_PINTERP,
111   OP_EMIT,    // emit vertex
112   OP_RESTART, // restart primitive
113   OP_TEX,
114   OP_TXB, // texture bias
115   OP_TXL, // texure lod
116   OP_TXF, // texel fetch
117   OP_TXQ, // texture size query
118   OP_TXD, // texture derivatives
119   OP_TXG, // texture gather
120   OP_TEXCSAA,
121   OP_SULD, // surface load
122   OP_SUST, // surface store
123   OP_DFDX,
124   OP_DFDY,
125   OP_RDSV, // read system value
126   OP_WRSV, // write system value
127   OP_PIXLD,
128   OP_QUADOP,
129   OP_QUADON,
130   OP_QUADPOP,
131   OP_POPCNT, // bitcount(src0 & src1)
132   OP_INSBF,  // insert first src1[8:15] bits of src0 into src2 at src1[0:7]
133   OP_EXTBF,
134   OP_TEXBAR,
135   OP_LAST
136};
137
138// various instruction-specific modifier definitions Instruction::subOp
139// MOV_FINAL marks a MOV originating from an EXPORT (used for placing TEXBARs)
140#define NV50_IR_SUBOP_MUL_HIGH     1
141#define NV50_IR_SUBOP_EMIT_RESTART 1
142#define NV50_IR_SUBOP_LDC_IL       1
143#define NV50_IR_SUBOP_LDC_IS       2
144#define NV50_IR_SUBOP_LDC_ISL      3
145#define NV50_IR_SUBOP_SHIFT_WRAP   1
146#define NV50_IR_SUBOP_EMU_PRERET   1
147#define NV50_IR_SUBOP_TEXBAR(n)    n
148#define NV50_IR_SUBOP_MOV_FINAL    1
149
150enum DataType
151{
152   TYPE_NONE,
153   TYPE_U8,
154   TYPE_S8,
155   TYPE_U16,
156   TYPE_S16,
157   TYPE_U32,
158   TYPE_S32,
159   TYPE_U64, // 64 bit operations are only lowered after register allocation
160   TYPE_S64,
161   TYPE_F16,
162   TYPE_F32,
163   TYPE_F64,
164   TYPE_B96,
165   TYPE_B128
166};
167
168enum CondCode
169{
170   CC_FL = 0,
171   CC_NEVER = CC_FL, // when used with FILE_FLAGS
172   CC_LT = 1,
173   CC_EQ = 2,
174   CC_NOT_P = CC_EQ, // when used with FILE_PREDICATE
175   CC_LE = 3,
176   CC_GT = 4,
177   CC_NE = 5,
178   CC_P  = CC_NE,
179   CC_GE = 6,
180   CC_TR = 7,
181   CC_ALWAYS = CC_TR,
182   CC_U  = 8,
183   CC_LTU = 9,
184   CC_EQU = 10,
185   CC_LEU = 11,
186   CC_GTU = 12,
187   CC_NEU = 13,
188   CC_GEU = 14,
189   CC_NO = 0x10,
190   CC_NC = 0x11,
191   CC_NS = 0x12,
192   CC_NA = 0x13,
193   CC_A  = 0x14,
194   CC_S  = 0x15,
195   CC_C  = 0x16,
196   CC_O  = 0x17
197};
198
199enum RoundMode
200{
201   ROUND_N, // nearest
202   ROUND_M, // towards -inf
203   ROUND_Z, // towards 0
204   ROUND_P, // towards +inf
205   ROUND_NI, // nearest integer
206   ROUND_MI, // to integer towards -inf
207   ROUND_ZI, // to integer towards 0
208   ROUND_PI, // to integer towards +inf
209};
210
211enum CacheMode
212{
213   CACHE_CA,            // cache at all levels
214   CACHE_WB = CACHE_CA, // cache write back
215   CACHE_CG,            // cache at global level
216   CACHE_CS,            // cache streaming
217   CACHE_CV,            // cache as volatile
218   CACHE_WT = CACHE_CV  // cache write-through
219};
220
221enum DataFile
222{
223   FILE_NULL = 0,
224   FILE_GPR,
225   FILE_PREDICATE,       // boolean predicate
226   FILE_FLAGS,           // zero/sign/carry/overflow bits
227   FILE_ADDRESS,
228   LAST_REGISTER_FILE = FILE_ADDRESS,
229   FILE_IMMEDIATE,
230   FILE_MEMORY_CONST,
231   FILE_SHADER_INPUT,
232   FILE_SHADER_OUTPUT,
233   FILE_MEMORY_GLOBAL,
234   FILE_MEMORY_SHARED,
235   FILE_MEMORY_LOCAL,
236   FILE_SYSTEM_VALUE,
237   DATA_FILE_COUNT
238};
239
240enum TexTarget
241{
242   TEX_TARGET_1D,
243   TEX_TARGET_2D,
244   TEX_TARGET_2D_MS,
245   TEX_TARGET_3D,
246   TEX_TARGET_CUBE,
247   TEX_TARGET_1D_SHADOW,
248   TEX_TARGET_2D_SHADOW,
249   TEX_TARGET_CUBE_SHADOW,
250   TEX_TARGET_1D_ARRAY,
251   TEX_TARGET_2D_ARRAY,
252   TEX_TARGET_2D_MS_ARRAY,
253   TEX_TARGET_CUBE_ARRAY,
254   TEX_TARGET_1D_ARRAY_SHADOW,
255   TEX_TARGET_2D_ARRAY_SHADOW,
256   TEX_TARGET_RECT,
257   TEX_TARGET_RECT_SHADOW,
258   TEX_TARGET_CUBE_ARRAY_SHADOW,
259   TEX_TARGET_BUFFER,
260   TEX_TARGET_COUNT
261};
262
263enum SVSemantic
264{
265   SV_POSITION, // WPOS
266   SV_VERTEX_ID,
267   SV_INSTANCE_ID,
268   SV_INVOCATION_ID,
269   SV_PRIMITIVE_ID,
270   SV_VERTEX_COUNT, // gl_PatchVerticesIn
271   SV_LAYER,
272   SV_VIEWPORT_INDEX,
273   SV_YDIR,
274   SV_FACE,
275   SV_POINT_SIZE,
276   SV_POINT_COORD,
277   SV_CLIP_DISTANCE,
278   SV_SAMPLE_INDEX,
279   SV_TESS_FACTOR,
280   SV_TESS_COORD,
281   SV_TID,
282   SV_CTAID,
283   SV_NTID,
284   SV_GRIDID,
285   SV_NCTAID,
286   SV_LANEID,
287   SV_PHYSID,
288   SV_NPHYSID,
289   SV_CLOCK,
290   SV_LBASE,
291   SV_SBASE,
292   SV_UNDEFINED,
293   SV_LAST
294};
295
296class Program;
297class Function;
298class BasicBlock;
299
300class Target;
301
302class Instruction;
303class CmpInstruction;
304class TexInstruction;
305class FlowInstruction;
306
307class Value;
308class LValue;
309class Symbol;
310class ImmediateValue;
311
312struct Storage
313{
314   DataFile file;
315   int8_t fileIndex; // signed, may be indirect for CONST[]
316   uint8_t size; // this should match the Instruction type's size
317   DataType type; // mainly for pretty printing
318   union {
319      uint64_t u64;    // immediate values
320      uint32_t u32;
321      uint16_t u16;
322      uint8_t u8;
323      int64_t s64;
324      int32_t s32;
325      int16_t s16;
326      int8_t s8;
327      float f32;
328      double f64;
329      int32_t offset; // offset from 0 (base of address space)
330      int32_t id;     // register id (< 0 if virtual/unassigned, in units <= 4)
331      struct {
332         SVSemantic sv;
333         int index;
334      } sv;
335   } data;
336};
337
338// precedence: NOT after SAT after NEG after ABS
339#define NV50_IR_MOD_ABS (1 << 0)
340#define NV50_IR_MOD_NEG (1 << 1)
341#define NV50_IR_MOD_SAT (1 << 2)
342#define NV50_IR_MOD_NOT (1 << 3)
343#define NV50_IR_MOD_NEG_ABS (NV50_IR_MOD_NEG | NV50_IR_MOD_ABS)
344
345#define NV50_IR_INTERP_MODE_MASK   0x3
346#define NV50_IR_INTERP_LINEAR      (0 << 0)
347#define NV50_IR_INTERP_PERSPECTIVE (1 << 0)
348#define NV50_IR_INTERP_FLAT        (2 << 0)
349#define NV50_IR_INTERP_SC          (3 << 0) // what exactly is that ?
350#define NV50_IR_INTERP_SAMPLE_MASK 0xc
351#define NV50_IR_INTERP_DEFAULT     (0 << 2)
352#define NV50_IR_INTERP_CENTROID    (1 << 2)
353#define NV50_IR_INTERP_OFFSET      (2 << 2)
354#define NV50_IR_INTERP_SAMPLEID    (3 << 2)
355
356// do we really want this to be a class ?
357class Modifier
358{
359public:
360   Modifier() : bits(0) { }
361   Modifier(unsigned int m) : bits(m) { }
362   Modifier(operation op);
363
364   // @return new Modifier applying a after b (asserts if unrepresentable)
365   Modifier operator*(const Modifier) const;
366   Modifier operator*=(const Modifier m) { *this = *this * m; return *this; }
367   Modifier operator==(const Modifier m) const { return m.bits == bits; }
368   Modifier operator!=(const Modifier m) const { return m.bits != bits; }
369
370   inline Modifier operator&(const Modifier m) const { return bits & m.bits; }
371   inline Modifier operator|(const Modifier m) const { return bits | m.bits; }
372   inline Modifier operator^(const Modifier m) const { return bits ^ m.bits; }
373
374   operation getOp() const;
375
376   inline int neg() const { return (bits & NV50_IR_MOD_NEG) ? 1 : 0; }
377   inline int abs() const { return (bits & NV50_IR_MOD_ABS) ? 1 : 0; }
378
379   inline operator bool() const { return bits ? true : false; }
380
381   void applyTo(ImmediateValue &imm) const;
382
383   int print(char *buf, size_t size) const;
384
385private:
386   uint8_t bits;
387};
388
389class ValueRef
390{
391public:
392   ValueRef(Value * = NULL);
393   ValueRef(const ValueRef&);
394   ~ValueRef();
395
396   inline bool exists() const { return value != NULL; }
397
398   void set(Value *);
399   void set(const ValueRef&);
400   inline Value *get() const { return value; }
401   inline Value *rep() const;
402
403   inline Instruction *getInsn() const { return insn; }
404   inline void setInsn(Instruction *inst) { insn = inst; }
405
406   inline bool isIndirect(int dim) const { return indirect[dim] >= 0; }
407   inline const ValueRef *getIndirect(int dim) const;
408
409   inline DataFile getFile() const;
410   inline unsigned getSize() const;
411
412   // SSA: return eventual (traverse MOVs) literal value, if it exists
413   bool getImmediate(ImmediateValue&) const;
414
415public:
416   Modifier mod;
417   int8_t indirect[2]; // >= 0 if relative to lvalue in insn->src(indirect[i])
418   uint8_t swizzle;
419
420   bool usedAsPtr; // for printing
421
422private:
423   Value *value;
424   Instruction *insn;
425};
426
427class ValueDef
428{
429public:
430   ValueDef(Value * = NULL);
431   ValueDef(const ValueDef&);
432   ~ValueDef();
433
434   inline bool exists() const { return value != NULL; }
435
436   inline Value *get() const { return value; }
437   inline Value *rep() const;
438   void set(Value *);
439   bool mayReplace(const ValueRef &);
440   void replace(const ValueRef &, bool doSet); // replace all uses of the old value
441
442   inline Instruction *getInsn() const { return insn; }
443   inline void setInsn(Instruction *inst) { insn = inst; }
444
445   inline DataFile getFile() const;
446   inline unsigned getSize() const;
447
448   inline void setSSA(LValue *);
449   inline const LValue *preSSA() const;
450
451private:
452   Value *value;   // should make this LValue * ...
453   LValue *origin; // pre SSA value
454   Instruction *insn;
455};
456
457class Value
458{
459public:
460   Value();
461   virtual ~Value() { }
462
463   virtual Value *clone(ClonePolicy<Function>&) const = 0;
464
465   virtual int print(char *, size_t, DataType ty = TYPE_NONE) const = 0;
466
467   virtual bool equals(const Value *, bool strict = false) const;
468   virtual bool interfers(const Value *) const;
469   virtual bool isUniform() const { return true; }
470
471   inline Value *rep() const { return join; }
472
473   inline Instruction *getUniqueInsn() const;
474   inline Instruction *getInsn() const; // use when uniqueness is certain
475
476   inline int refCount() { return uses.size(); }
477
478   inline LValue *asLValue();
479   inline Symbol *asSym();
480   inline ImmediateValue *asImm();
481   inline const Symbol *asSym() const;
482   inline const ImmediateValue *asImm() const;
483
484   inline bool inFile(DataFile f) { return reg.file == f; }
485
486   static inline Value *get(Iterator&);
487
488   std::list<ValueRef *> uses;
489   std::list<ValueDef *> defs;
490   typedef std::list<ValueRef *>::iterator UseIterator;
491   typedef std::list<ValueRef *>::const_iterator UseCIterator;
492   typedef std::list<ValueDef *>::iterator DefIterator;
493   typedef std::list<ValueDef *>::const_iterator DefCIterator;
494
495   int id;
496   Storage reg;
497
498   // TODO: these should be in LValue:
499   Interval livei;
500   Value *join;
501};
502
503class LValue : public Value
504{
505public:
506   LValue(Function *, DataFile file);
507   LValue(Function *, LValue *);
508   ~LValue() { }
509
510   virtual bool isUniform() const;
511
512   virtual LValue *clone(ClonePolicy<Function>&) const;
513
514   virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
515
516public:
517   unsigned compMask : 8; // compound/component mask
518   unsigned compound : 1; // used by RA, value involved in split/merge
519   unsigned ssa      : 1;
520   unsigned fixedReg : 1; // set & used by RA, earlier just use (id < 0)
521   unsigned noSpill  : 1; // do not spill (e.g. if spill temporary already)
522};
523
524class Symbol : public Value
525{
526public:
527   Symbol(Program *, DataFile file = FILE_MEMORY_CONST, ubyte fileIdx = 0);
528   ~Symbol() { }
529
530   virtual Symbol *clone(ClonePolicy<Function>&) const;
531
532   virtual bool equals(const Value *that, bool strict) const;
533
534   virtual bool isUniform() const;
535
536   virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
537
538   // print with indirect values
539   int print(char *, size_t, Value *, Value *, DataType ty = TYPE_NONE) const;
540
541   inline void setFile(DataFile file, ubyte fileIndex = 0)
542   {
543      reg.file = file;
544      reg.fileIndex = fileIndex;
545   }
546
547   inline void setOffset(int32_t offset);
548   inline void setAddress(Symbol *base, int32_t offset);
549   inline void setSV(SVSemantic sv, uint32_t idx = 0);
550
551   inline const Symbol *getBase() const { return baseSym; }
552
553private:
554   Symbol *baseSym; // array base for Symbols representing array elements
555};
556
557class ImmediateValue : public Value
558{
559public:
560   ImmediateValue() { }
561   ImmediateValue(Program *, uint32_t);
562   ImmediateValue(Program *, float);
563   ImmediateValue(Program *, double);
564   // NOTE: not added to program with
565   ImmediateValue(const ImmediateValue *, DataType ty);
566   ~ImmediateValue() { };
567
568   virtual ImmediateValue *clone(ClonePolicy<Function>&) const;
569
570   virtual bool equals(const Value *that, bool strict) const;
571
572   // these only work if 'type' is valid (we mostly use untyped literals):
573   bool isInteger(const int ival) const; // ival is cast to this' type
574   bool isNegative() const;
575   bool isPow2() const;
576
577   void applyLog2();
578
579   // for constant folding:
580   ImmediateValue operator+(const ImmediateValue&) const;
581   ImmediateValue operator-(const ImmediateValue&) const;
582   ImmediateValue operator*(const ImmediateValue&) const;
583   ImmediateValue operator/(const ImmediateValue&) const;
584
585   ImmediateValue& operator=(const ImmediateValue&); // only sets value !
586
587   bool compare(CondCode cc, float fval) const;
588
589   virtual int print(char *, size_t, DataType ty = TYPE_NONE) const;
590};
591
592class Instruction
593{
594public:
595   Instruction();
596   Instruction(Function *, operation, DataType);
597   virtual ~Instruction();
598
599   virtual Instruction *clone(ClonePolicy<Function>&,
600                              Instruction * = NULL) const;
601
602   void setDef(int i, Value *);
603   void setSrc(int s, Value *);
604   void setSrc(int s, const ValueRef&);
605   void swapSources(int a, int b);
606   void moveSources(int s, int delta); // NOTE: only delta > 0 implemented
607   bool setIndirect(int s, int dim, Value *);
608
609   inline ValueRef& src(int s) { return srcs[s]; }
610   inline ValueDef& def(int s) { return defs[s]; }
611   inline const ValueRef& src(int s) const { return srcs[s]; }
612   inline const ValueDef& def(int s) const { return defs[s]; }
613
614   inline Value *getDef(int d) const { return defs[d].get(); }
615   inline Value *getSrc(int s) const { return srcs[s].get(); }
616   inline Value *getIndirect(int s, int dim) const;
617
618   inline bool defExists(unsigned d) const
619   {
620      return d < defs.size() && defs[d].exists();
621   }
622   inline bool srcExists(unsigned s) const
623   {
624      return s < srcs.size() && srcs[s].exists();
625   }
626
627   inline bool constrainedDefs() const;
628
629   bool setPredicate(CondCode ccode, Value *);
630   inline Value *getPredicate() const;
631   bool writesPredicate() const;
632   inline bool isPredicated() const { return predSrc >= 0; }
633
634   inline void setFlagsSrc(int s, Value *);
635   inline void setFlagsDef(int d, Value *);
636
637   unsigned int defCount() const { return defs.size(); };
638   unsigned int defCount(unsigned int mask, bool singleFile = false) const;
639   unsigned int srcCount() const { return srcs.size(); };
640   unsigned int srcCount(unsigned int mask, bool singleFile = false) const;
641
642   // save & remove / set indirect[0,1] and predicate source
643   void takeExtraSources(int s, Value *[3]);
644   void putExtraSources(int s, Value *[3]);
645
646   inline void setType(DataType type) { dType = sType = type; }
647
648   inline void setType(DataType dtype, DataType stype)
649   {
650      dType = dtype;
651      sType = stype;
652   }
653
654   inline bool isPseudo() const { return op < OP_MOV; }
655   bool isDead() const;
656   bool isNop() const;
657   bool isCommutationLegal(const Instruction *) const; // must be adjacent !
658   bool isActionEqual(const Instruction *) const;
659   bool isResultEqual(const Instruction *) const;
660
661   void print() const;
662
663   inline CmpInstruction *asCmp();
664   inline TexInstruction *asTex();
665   inline FlowInstruction *asFlow();
666   inline const TexInstruction *asTex() const;
667   inline const CmpInstruction *asCmp() const;
668   inline const FlowInstruction *asFlow() const;
669
670public:
671   Instruction *next;
672   Instruction *prev;
673   int id;
674   int serial; // CFG order
675
676   operation op;
677   DataType dType; // destination or defining type
678   DataType sType; // source or secondary type
679   CondCode cc;
680   RoundMode rnd;
681   CacheMode cache;
682
683   uint8_t subOp; // quadop, 1 for mul-high, etc.
684
685   uint8_t sched; // scheduling data (NOTE: maybe move to separate storage)
686
687   unsigned encSize    : 4; // encoding size in bytes
688   unsigned saturate   : 1; // to [0.0f, 1.0f]
689   unsigned join       : 1; // converge control flow (use OP_JOIN until end)
690   unsigned fixed      : 1; // prevent dead code elimination
691   unsigned terminator : 1; // end of basic block
692   unsigned atomic     : 1;
693   unsigned ftz        : 1; // flush denormal to zero
694   unsigned dnz        : 1; // denormals, NaN are zero
695   unsigned ipa        : 4; // interpolation mode
696   unsigned lanes      : 4;
697   unsigned perPatch   : 1;
698   unsigned exit       : 1; // terminate program after insn
699
700   int8_t postFactor; // MUL/DIV(if < 0) by 1 << postFactor
701
702   int8_t predSrc;
703   int8_t flagsDef;
704   int8_t flagsSrc;
705
706   BasicBlock *bb;
707
708protected:
709   std::deque<ValueDef> defs; // no gaps !
710   std::deque<ValueRef> srcs; // no gaps !
711
712   // instruction specific methods:
713   // (don't want to subclass, would need more constructors and memory pools)
714public:
715   inline void setInterpolate(unsigned int mode) { ipa = mode; }
716
717   unsigned int getInterpMode() const { return ipa & 0x3; }
718   unsigned int getSampleMode() const { return ipa & 0xc; }
719
720private:
721   void init();
722};
723
724enum TexQuery
725{
726   TXQ_DIMS,
727   TXQ_TYPE,
728   TXQ_SAMPLE_POSITION,
729   TXQ_FILTER,
730   TXQ_LOD,
731   TXQ_WRAP,
732   TXQ_BORDER_COLOUR
733};
734
735class TexInstruction : public Instruction
736{
737public:
738   class Target
739   {
740   public:
741      Target(TexTarget targ = TEX_TARGET_2D) : target(targ) { }
742
743      const char *getName() const { return descTable[target].name; }
744      unsigned int getArgCount() const { return descTable[target].argc; }
745      unsigned int getDim() const { return descTable[target].dim; }
746      int isArray() const { return descTable[target].array ? 1 : 0; }
747      int isCube() const { return descTable[target].cube ? 1 : 0; }
748      int isShadow() const { return descTable[target].shadow ? 1 : 0; }
749
750      Target& operator=(TexTarget targ)
751      {
752         assert(targ < TEX_TARGET_COUNT);
753         return *this;
754      }
755
756      inline bool operator==(TexTarget targ) const { return target == targ; }
757
758   private:
759      struct Desc
760      {
761         char name[19];
762         uint8_t dim;
763         uint8_t argc;
764         bool array;
765         bool cube;
766         bool shadow;
767      };
768
769      static const struct Desc descTable[TEX_TARGET_COUNT];
770
771   private:
772      enum TexTarget target;
773   };
774
775public:
776   TexInstruction(Function *, operation);
777   virtual ~TexInstruction();
778
779   virtual TexInstruction *clone(ClonePolicy<Function>&,
780                                 Instruction * = NULL) const;
781
782   inline void setTexture(Target targ, uint8_t r, uint8_t s)
783   {
784      tex.r = r;
785      tex.s = s;
786      tex.target = targ;
787   }
788
789   inline Value *getIndirectR() const;
790   inline Value *getIndirectS() const;
791
792public:
793   struct {
794      Target target;
795
796      uint8_t r;
797      int8_t rIndirectSrc;
798      uint8_t s;
799      int8_t sIndirectSrc;
800
801      uint8_t mask;
802      uint8_t gatherComp;
803
804      bool liveOnly; // only execute on live pixels of a quad (optimization)
805      bool levelZero;
806      bool derivAll;
807
808      int8_t useOffsets; // 0, 1, or 4 for textureGatherOffsets
809      int8_t offset[4][3];
810
811      enum TexQuery query;
812   } tex;
813
814   ValueRef dPdx[3];
815   ValueRef dPdy[3];
816};
817
818class CmpInstruction : public Instruction
819{
820public:
821   CmpInstruction(Function *, operation);
822
823   virtual CmpInstruction *clone(ClonePolicy<Function>&,
824                                 Instruction * = NULL) const;
825
826   void setCondition(CondCode cond) { setCond = cond; }
827   CondCode getCondition() const { return setCond; }
828
829public:
830   CondCode setCond;
831};
832
833class FlowInstruction : public Instruction
834{
835public:
836   FlowInstruction(Function *, operation, void *target);
837
838   virtual FlowInstruction *clone(ClonePolicy<Function>&,
839                                  Instruction * = NULL) const;
840
841public:
842   unsigned allWarp  : 1;
843   unsigned absolute : 1;
844   unsigned limit    : 1;
845   unsigned builtin  : 1; // true for calls to emulation code
846
847   union {
848      BasicBlock *bb;
849      int builtin;
850      Function *fn;
851   } target;
852};
853
854class BasicBlock
855{
856public:
857   BasicBlock(Function *);
858   ~BasicBlock();
859
860   BasicBlock *clone(ClonePolicy<Function>&) const;
861
862   inline int getId() const { return id; }
863   inline unsigned int getInsnCount() const { return numInsns; }
864   inline bool isTerminated() const { return exit && exit->terminator; }
865
866   bool dominatedBy(BasicBlock *bb);
867   inline bool reachableBy(const BasicBlock *by, const BasicBlock *term);
868
869   // returns mask of conditional out blocks
870   // e.g. 3 for IF { .. } ELSE { .. } ENDIF, 1 for IF { .. } ENDIF
871   unsigned int initiatesSimpleConditional() const;
872
873public:
874   Function *getFunction() const { return func; }
875   Program *getProgram() const { return program; }
876
877   Instruction *getEntry() const { return entry; } // first non-phi instruction
878   Instruction *getPhi() const { return phi; }
879   Instruction *getFirst() const { return phi ? phi : entry; }
880   Instruction *getExit() const { return exit; }
881
882   void insertHead(Instruction *);
883   void insertTail(Instruction *);
884   void insertBefore(Instruction *, Instruction *);
885   void insertAfter(Instruction *, Instruction *);
886   void remove(Instruction *);
887   void permuteAdjacent(Instruction *, Instruction *);
888
889   BasicBlock *idom() const;
890
891   // NOTE: currently does not rebuild the dominator tree
892   BasicBlock *splitBefore(Instruction *, bool attach = true);
893   BasicBlock *splitAfter(Instruction *, bool attach = true);
894
895   DLList& getDF() { return df; }
896   DLList::Iterator iterDF() { return df.iterator(); }
897
898   static inline BasicBlock *get(Iterator&);
899   static inline BasicBlock *get(Graph::Node *);
900
901public:
902   Graph::Node cfg; // first edge is branch *taken* (the ELSE branch)
903   Graph::Node dom;
904
905   BitSet liveSet;
906   BitSet defSet;
907
908   uint32_t binPos;
909   uint32_t binSize;
910
911   Instruction *joinAt; // for quick reference
912
913   bool explicitCont; // loop headers: true if loop contains continue stmts
914
915private:
916   int id;
917   DLList df;
918
919   Instruction *phi;
920   Instruction *entry;
921   Instruction *exit;
922
923   unsigned int numInsns;
924
925private:
926   Function *func;
927   Program *program;
928
929   void splitCommon(Instruction *, BasicBlock *, bool attach);
930};
931
932class Function
933{
934public:
935   Function(Program *, const char *name, uint32_t label);
936   ~Function();
937
938   static inline Function *get(Graph::Node *node);
939
940   inline Program *getProgram() const { return prog; }
941   inline const char *getName() const { return name; }
942   inline int getId() const { return id; }
943   inline uint32_t getLabel() const { return label; }
944
945   void print();
946   void printLiveIntervals() const;
947   void printCFGraph(const char *filePath);
948
949   bool setEntry(BasicBlock *);
950   bool setExit(BasicBlock *);
951
952   unsigned int orderInstructions(ArrayList&);
953
954   inline void add(BasicBlock *bb, int& id) { allBBlocks.insert(bb, id); }
955   inline void add(Instruction *insn, int& id) { allInsns.insert(insn, id); }
956   inline void add(LValue *lval, int& id) { allLValues.insert(lval, id); }
957
958   inline LValue *getLValue(int id);
959
960   void buildLiveSets();
961   void buildDefSets();
962   bool convertToSSA();
963
964public:
965   std::deque<ValueDef> ins;
966   std::deque<ValueRef> outs;
967   std::deque<Value *> clobbers;
968
969   Graph cfg;
970   Graph::Node *cfgExit;
971   Graph *domTree;
972   Graph::Node call; // node in the call graph
973
974   BasicBlock **bbArray; // BBs in emission order
975   int bbCount;
976
977   unsigned int loopNestingBound;
978   int regClobberMax;
979
980   uint32_t binPos;
981   uint32_t binSize;
982
983   Value *stackPtr;
984
985   uint32_t tlsBase; // base address for l[] space (if no stack pointer is used)
986   uint32_t tlsSize;
987
988   ArrayList allBBlocks;
989   ArrayList allInsns;
990   ArrayList allLValues;
991
992private:
993   void buildLiveSetsPreSSA(BasicBlock *, const int sequence);
994   void buildDefSetsPreSSA(BasicBlock *bb, const int seq);
995
996private:
997   uint32_t label;
998   int id;
999   const char *const name;
1000   Program *prog;
1001};
1002
1003enum CGStage
1004{
1005   CG_STAGE_PRE_SSA,
1006   CG_STAGE_SSA, // expected directly before register allocation
1007   CG_STAGE_POST_RA
1008};
1009
1010class Program
1011{
1012public:
1013   enum Type
1014   {
1015      TYPE_VERTEX,
1016      TYPE_TESSELLATION_CONTROL,
1017      TYPE_TESSELLATION_EVAL,
1018      TYPE_GEOMETRY,
1019      TYPE_FRAGMENT,
1020      TYPE_COMPUTE
1021   };
1022
1023   Program(Type type, Target *targ);
1024   ~Program();
1025
1026   void print();
1027
1028   Type getType() const { return progType; }
1029
1030   inline void add(Function *fn, int& id) { allFuncs.insert(fn, id); }
1031   inline void del(Function *fn, int& id) { allFuncs.remove(id); }
1032   inline void add(Value *rval, int& id) { allRValues.insert(rval, id); }
1033
1034   bool makeFromTGSI(struct nv50_ir_prog_info *);
1035   bool makeFromSM4(struct nv50_ir_prog_info *);
1036   bool convertToSSA();
1037   bool optimizeSSA(int level);
1038   bool optimizePostRA(int level);
1039   bool registerAllocation();
1040   bool emitBinary(struct nv50_ir_prog_info *);
1041
1042   const Target *getTarget() const { return target; }
1043
1044private:
1045   void emitSymbolTable(struct nv50_ir_prog_info *);
1046
1047   Type progType;
1048   Target *target;
1049
1050public:
1051   Function *main;
1052   Graph calls;
1053
1054   ArrayList allFuncs;
1055   ArrayList allRValues;
1056
1057   uint32_t *code;
1058   uint32_t binSize;
1059   uint32_t tlsSize; // size required for FILE_MEMORY_LOCAL
1060
1061   int maxGPR;
1062
1063   MemoryPool mem_Instruction;
1064   MemoryPool mem_CmpInstruction;
1065   MemoryPool mem_TexInstruction;
1066   MemoryPool mem_FlowInstruction;
1067   MemoryPool mem_LValue;
1068   MemoryPool mem_Symbol;
1069   MemoryPool mem_ImmediateValue;
1070
1071   uint32_t dbgFlags;
1072   uint8_t  optLevel;
1073
1074   void *targetPriv; // e.g. to carry information between passes
1075
1076   void releaseInstruction(Instruction *);
1077   void releaseValue(Value *);
1078};
1079
1080// TODO: add const version
1081class Pass
1082{
1083public:
1084   bool run(Program *, bool ordered = false, bool skipPhi = false);
1085   bool run(Function *, bool ordered = false, bool skipPhi = false);
1086
1087private:
1088   // return false to continue with next entity on next higher level
1089   virtual bool visit(Function *) { return true; }
1090   virtual bool visit(BasicBlock *) { return true; }
1091   virtual bool visit(Instruction *) { return false; }
1092
1093   bool doRun(Program *, bool ordered, bool skipPhi);
1094   bool doRun(Function *, bool ordered, bool skipPhi);
1095
1096protected:
1097   bool err;
1098   Function *func;
1099   Program *prog;
1100};
1101
1102// =============================================================================
1103
1104#include "nv50_ir_inlines.h"
1105
1106} // namespace nv50_ir
1107
1108#endif // __NV50_IR_H__
1109