compiler/nir/nir_search.c

/*
 * Copyright © 2014 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 *
 * Authors:
 *    Jason Ekstrand (jason@jlekstrand.net)
 *
 */

#include <inttypes.h>
#include "nir_search.h"

struct match_state {
   bool inexact_match;
   bool has_exact_alu;
   unsigned variables_seen;
   nir_alu_src variables[NIR_SEARCH_MAX_VARIABLES];
};

static bool
match_expression(const nir_search_expression *expr, nir_alu_instr *instr,
                 unsigned num_components, const uint8_t *swizzle,
                 struct match_state *state);

static const uint8_t identity_swizzle[] = { 0, 1, 2, 3 };

/**
 * Check if a source produces a value of the given type.
 *
 * Used for satisfying 'a@type' constraints.
 */
static bool
src_is_type(nir_src src, nir_alu_type type)
{
   assert(type != nir_type_invalid);

   if (!src.is_ssa)
      return false;

   /* Turn nir_type_bool32 into nir_type_bool...they're the same thing. */
   if (nir_alu_type_get_base_type(type) == nir_type_bool)
      type = nir_type_bool;

   if (src.ssa->parent_instr->type == nir_instr_type_alu) {
      nir_alu_instr *src_alu = nir_instr_as_alu(src.ssa->parent_instr);
      nir_alu_type output_type = nir_op_infos[src_alu->op].output_type;

      if (type == nir_type_bool) {
         switch (src_alu->op) {
         case nir_op_iand:
         case nir_op_ior:
         case nir_op_ixor:
            return src_is_type(src_alu->src[0].src, nir_type_bool) &&
                   src_is_type(src_alu->src[1].src, nir_type_bool);
         case nir_op_inot:
            return src_is_type(src_alu->src[0].src, nir_type_bool);
         default:
            break;
         }
      }

      return nir_alu_type_get_base_type(output_type) == type;
   } else if (src.ssa->parent_instr->type == nir_instr_type_intrinsic) {
      nir_intrinsic_instr *intr = nir_instr_as_intrinsic(src.ssa->parent_instr);

      if (type == nir_type_bool) {
         return intr->intrinsic == nir_intrinsic_load_front_face ||
                intr->intrinsic == nir_intrinsic_load_helper_invocation;
      }
   }

   /* don't know */
   return false;
}

static bool
match_value(const nir_search_value *value, nir_alu_instr *instr, unsigned src,
            unsigned num_components, const uint8_t *swizzle,
            struct match_state *state)
{
   uint8_t new_swizzle[4];

   /* Searching only works on SSA values because, if it's not SSA, we can't
    * know if the value changed between one instance of that value in the
    * expression and another.  Also, the replace operation will place reads of
    * that value right before the last instruction in the expression we're
    * replacing so those reads will happen after the original reads and may
    * not be valid if they're register reads.
    */
   if (!instr->src[src].src.is_ssa)
      return false;

   /* If the source is an explicitly sized source, then we need to reset
    * both the number of components and the swizzle.
    */
   if (nir_op_infos[instr->op].input_sizes[src] != 0) {
      num_components = nir_op_infos[instr->op].input_sizes[src];
      swizzle = identity_swizzle;
   }

   for (unsigned i = 0; i < num_components; ++i)
      new_swizzle[i] = instr->src[src].swizzle[swizzle[i]];

   /* If the value has a specific bit size and it doesn't match, bail */
   if (value->bit_size &&
       nir_src_bit_size(instr->src[src].src) != value->bit_size)
      return false;

   switch (value->type) {
   case nir_search_value_expression:
      if (instr->src[src].src.ssa->parent_instr->type != nir_instr_type_alu)
         return false;

      return match_expression(nir_search_value_as_expression(value),
                              nir_instr_as_alu(instr->src[src].src.ssa->parent_instr),
                              num_components, new_swizzle, state);

   case nir_search_value_variable: {
      nir_search_variable *var = nir_search_value_as_variable(value);
      assert(var->variable < NIR_SEARCH_MAX_VARIABLES);

      if (state->variables_seen & (1 << var->variable)) {
         if (state->variables[var->variable].src.ssa != instr->src[src].src.ssa)
            return false;

         assert(!instr->src[src].abs && !instr->src[src].negate);

         for (unsigned i = 0; i < num_components; ++i) {
            if (state->variables[var->variable].swizzle[i] != new_swizzle[i])
               return false;
         }

         return true;
      } else {
         if (var->is_constant &&
             instr->src[src].src.ssa->parent_instr->type != nir_instr_type_load_const)
            return false;

         if (var->cond && !var->cond(instr, src, num_components, new_swizzle))
            return false;

         if (var->type != nir_type_invalid &&
             !src_is_type(instr->src[src].src, var->type))
            return false;

         state->variables_seen |= (1 << var->variable);
         state->variables[var->variable].src = instr->src[src].src;
         state->variables[var->variable].abs = false;
         state->variables[var->variable].negate = false;

         for (unsigned i = 0; i < 4; ++i) {
            if (i < num_components)
               state->variables[var->variable].swizzle[i] = new_swizzle[i];
            else
               state->variables[var->variable].swizzle[i] = 0;
         }

         return true;
      }
   }

   case nir_search_value_constant: {
      nir_search_constant *const_val = nir_search_value_as_constant(value);

      if (!instr->src[src].src.is_ssa)
         return false;

      if (instr->src[src].src.ssa->parent_instr->type != nir_instr_type_load_const)
         return false;

      nir_load_const_instr *load =
         nir_instr_as_load_const(instr->src[src].src.ssa->parent_instr);

      switch (const_val->type) {
      case nir_type_float:
         for (unsigned i = 0; i < num_components; ++i) {
            double val;
            switch (load->def.bit_size) {
            case 32:
               val = load->value.f32[new_swizzle[i]];
               break;
            case 64:
               val = load->value.f64[new_swizzle[i]];
               break;
            default:
               unreachable("unknown bit size");
            }

            if (val != const_val->data.d)
               return false;
         }
         return true;

      case nir_type_int:
         for (unsigned i = 0; i < num_components; ++i) {
            int64_t val;
            switch (load->def.bit_size) {
            case 32:
               val = load->value.i32[new_swizzle[i]];
               break;
            case 64:
               val = load->value.i64[new_swizzle[i]];
               break;
            default:
               unreachable("unknown bit size");
            }

            if (val != const_val->data.i)
               return false;
         }
         return true;

      case nir_type_uint:
      case nir_type_bool32:
         for (unsigned i = 0; i < num_components; ++i) {
            uint64_t val;
            switch (load->def.bit_size) {
            case 32:
               val = load->value.u32[new_swizzle[i]];
               break;
            case 64:
               val = load->value.u64[new_swizzle[i]];
               break;
            default:
               unreachable("unknown bit size");
            }

            if (val != const_val->data.u)
               return false;
         }
         return true;

      default:
         unreachable("Invalid alu source type");
      }
   }

   default:
      unreachable("Invalid search value type");
   }
}

static bool
match_expression(const nir_search_expression *expr, nir_alu_instr *instr,
                 unsigned num_components, const uint8_t *swizzle,
                 struct match_state *state)
{
   if (expr->cond && !expr->cond(instr))
      return false;

   if (instr->op != expr->opcode)
      return false;

   assert(instr->dest.dest.is_ssa);

   if (expr->value.bit_size &&
       instr->dest.dest.ssa.bit_size != expr->value.bit_size)
      return false;

   state->inexact_match = expr->inexact || state->inexact_match;
   state->has_exact_alu = instr->exact || state->has_exact_alu;
   if (state->inexact_match && state->has_exact_alu)
      return false;

   assert(!instr->dest.saturate);
   assert(nir_op_infos[instr->op].num_inputs > 0);

   /* If we have an explicitly sized destination, we can only handle the
    * identity swizzle.  While dot(vec3(a, b, c).zxy) is a valid
    * expression, we don't have the information right now to propagate that
    * swizzle through.  We can only properly propagate swizzles if the
    * instruction is vectorized.
    */
   if (nir_op_infos[instr->op].output_size != 0) {
      for (unsigned i = 0; i < num_components; i++) {
         if (swizzle[i] != i)
            return false;
      }
   }

   /* Stash off the current variables_seen bitmask.  This way we can
    * restore it prior to matching in the commutative case below.
    */
   unsigned variables_seen_stash = state->variables_seen;

   bool matched = true;
   for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
      if (!match_value(expr->srcs[i], instr, i, num_components,
                       swizzle, state)) {
         matched = false;
         break;
      }
   }

   if (matched)
      return true;

   if (nir_op_infos[instr->op].algebraic_properties & NIR_OP_IS_COMMUTATIVE) {
      assert(nir_op_infos[instr->op].num_inputs == 2);

      /* Restore the variables_seen bitmask.  If we don't do this, then we
       * could end up with an erroneous failure due to variables found in the
       * first match attempt above not matching those in the second.
       */
      state->variables_seen = variables_seen_stash;

      if (!match_value(expr->srcs[0], instr, 1, num_components,
                       swizzle, state))
         return false;

      return match_value(expr->srcs[1], instr, 0, num_components,
                         swizzle, state);
   } else {
      return false;
   }
}

typedef struct bitsize_tree {
   unsigned num_srcs;
   struct bitsize_tree *srcs[4];

   unsigned common_size;
   bool is_src_sized[4];
   bool is_dest_sized;

   unsigned dest_size;
   unsigned src_size[4];
} bitsize_tree;

static bitsize_tree *
build_bitsize_tree(void *mem_ctx, struct match_state *state,
                   const nir_search_value *value)
{
   bitsize_tree *tree = rzalloc(mem_ctx, bitsize_tree);

   switch (value->type) {
   case nir_search_value_expression: {
      nir_search_expression *expr = nir_search_value_as_expression(value);
      nir_op_info info = nir_op_infos[expr->opcode];
      tree->num_srcs = info.num_inputs;
      tree->common_size = 0;
      for (unsigned i = 0; i < info.num_inputs; i++) {
         tree->is_src_sized[i] = !!nir_alu_type_get_type_size(info.input_types[i]);
         if (tree->is_src_sized[i])
            tree->src_size[i] = nir_alu_type_get_type_size(info.input_types[i]);
         tree->srcs[i] = build_bitsize_tree(mem_ctx, state, expr->srcs[i]);
      }
      tree->is_dest_sized = !!nir_alu_type_get_type_size(info.output_type);
      if (tree->is_dest_sized)
         tree->dest_size = nir_alu_type_get_type_size(info.output_type);
      break;
   }

   case nir_search_value_variable: {
      nir_search_variable *var = nir_search_value_as_variable(value);
      tree->num_srcs = 0;
      tree->is_dest_sized = true;
      tree->dest_size = nir_src_bit_size(state->variables[var->variable].src);
      break;
   }

   case nir_search_value_constant: {
      tree->num_srcs = 0;
      tree->is_dest_sized = false;
      tree->common_size = 0;
      break;
   }
   }

   if (value->bit_size) {
      assert(!tree->is_dest_sized || tree->dest_size == value->bit_size);
      tree->common_size = value->bit_size;
   }

   return tree;
}

static unsigned
bitsize_tree_filter_up(bitsize_tree *tree)
{
   for (unsigned i = 0; i < tree->num_srcs; i++) {
      unsigned src_size = bitsize_tree_filter_up(tree->srcs[i]);
      if (src_size == 0)
         continue;

      if (tree->is_src_sized[i]) {
         assert(src_size == tree->src_size[i]);
      } else if (tree->common_size != 0) {
         assert(src_size == tree->common_size);
         tree->src_size[i] = src_size;
      } else {
         tree->common_size = src_size;
         tree->src_size[i] = src_size;
      }
   }

   if (tree->num_srcs && tree->common_size) {
      if (tree->dest_size == 0)
         tree->dest_size = tree->common_size;
      else if (!tree->is_dest_sized)
         assert(tree->dest_size == tree->common_size);

      for (unsigned i = 0; i < tree->num_srcs; i++) {
         if (!tree->src_size[i])
            tree->src_size[i] = tree->common_size;
      }
   }

   return tree->dest_size;
}

static void
bitsize_tree_filter_down(bitsize_tree *tree, unsigned size)
{
   if (tree->dest_size)
      assert(tree->dest_size == size);
   else
      tree->dest_size = size;

   if (!tree->is_dest_sized) {
      if (tree->common_size)
         assert(tree->common_size == size);
      else
         tree->common_size = size;
   }

   for (unsigned i = 0; i < tree->num_srcs; i++) {
      if (!tree->src_size[i]) {
         assert(tree->common_size);
         tree->src_size[i] = tree->common_size;
      }
      bitsize_tree_filter_down(tree->srcs[i], tree->src_size[i]);
   }
}

static nir_alu_src
construct_value(const nir_search_value *value,
                unsigned num_components, bitsize_tree *bitsize,
                struct match_state *state,
                nir_instr *instr, void *mem_ctx)
{
   switch (value->type) {
   case nir_search_value_expression: {
      const nir_search_expression *expr = nir_search_value_as_expression(value);

      if (nir_op_infos[expr->opcode].output_size != 0)
         num_components = nir_op_infos[expr->opcode].output_size;

      nir_alu_instr *alu = nir_alu_instr_create(mem_ctx, expr->opcode);
      nir_ssa_dest_init(&alu->instr, &alu->dest.dest, num_components,
                        bitsize->dest_size, NULL);
      alu->dest.write_mask = (1 << num_components) - 1;
      alu->dest.saturate = false;

      /* We have no way of knowing what values in a given search expression
       * map to a particular replacement value.  Therefore, if the
       * expression we are replacing has any exact values, the entire
       * replacement should be exact.
       */
      alu->exact = state->has_exact_alu;

      for (unsigned i = 0; i < nir_op_infos[expr->opcode].num_inputs; i++) {
         /* If the source is an explicitly sized source, then we need to reset
          * the number of components to match.
          */
         if (nir_op_infos[alu->op].input_sizes[i] != 0)
            num_components = nir_op_infos[alu->op].input_sizes[i];

         alu->src[i] = construct_value(expr->srcs[i],
                                       num_components, bitsize->srcs[i],
                                       state, instr, mem_ctx);
      }

      nir_instr_insert_before(instr, &alu->instr);

      nir_alu_src val;
      val.src = nir_src_for_ssa(&alu->dest.dest.ssa);
      val.negate = false;
      val.abs = false,
      memcpy(val.swizzle, identity_swizzle, sizeof val.swizzle);

      return val;
   }

   case nir_search_value_variable: {
      const nir_search_variable *var = nir_search_value_as_variable(value);
      assert(state->variables_seen & (1 << var->variable));

      nir_alu_src val = { NIR_SRC_INIT };
      nir_alu_src_copy(&val, &state->variables[var->variable], mem_ctx);

      assert(!var->is_constant);

      return val;
   }

   case nir_search_value_constant: {
      const nir_search_constant *c = nir_search_value_as_constant(value);
      nir_load_const_instr *load =
         nir_load_const_instr_create(mem_ctx, 1, bitsize->dest_size);

      switch (c->type) {
      case nir_type_float:
         load->def.name = ralloc_asprintf(load, "%f", c->data.d);
         switch (bitsize->dest_size) {
         case 32:
            load->value.f32[0] = c->data.d;
            break;
         case 64:
            load->value.f64[0] = c->data.d;
            break;
         default:
            unreachable("unknown bit size");
         }
         break;

      case nir_type_int:
         load->def.name = ralloc_asprintf(load, "%" PRIi64, c->data.i);
         switch (bitsize->dest_size) {
         case 32:
            load->value.i32[0] = c->data.i;
            break;
         case 64:
            load->value.i64[0] = c->data.i;
            break;
         default:
            unreachable("unknown bit size");
         }
         break;

      case nir_type_uint:
         load->def.name = ralloc_asprintf(load, "%" PRIu64, c->data.u);
         switch (bitsize->dest_size) {
         case 32:
            load->value.u32[0] = c->data.u;
            break;
         case 64:
            load->value.u64[0] = c->data.u;
            break;
         default:
            unreachable("unknown bit size");
         }
         break;

      case nir_type_bool32:
         load->value.u32[0] = c->data.u;
         break;
      default:
         unreachable("Invalid alu source type");
      }

      nir_instr_insert_before(instr, &load->instr);

      nir_alu_src val;
      val.src = nir_src_for_ssa(&load->def);
      val.negate = false;
      val.abs = false,
      memset(val.swizzle, 0, sizeof val.swizzle);

      return val;
   }

   default:
      unreachable("Invalid search value type");
   }
}

nir_alu_instr *
nir_replace_instr(nir_alu_instr *instr, const nir_search_expression *search,
                  const nir_search_value *replace, void *mem_ctx)
{
   uint8_t swizzle[4] = { 0, 0, 0, 0 };

   for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i)
      swizzle[i] = i;

   assert(instr->dest.dest.is_ssa);

   struct match_state state;
   state.inexact_match = false;
   state.has_exact_alu = false;
   state.variables_seen = 0;

   if (!match_expression(search, instr, instr->dest.dest.ssa.num_components,
                         swizzle, &state))
      return NULL;

   void *bitsize_ctx = ralloc_context(NULL);
   bitsize_tree *tree = build_bitsize_tree(bitsize_ctx, &state, replace);
   bitsize_tree_filter_up(tree);
   bitsize_tree_filter_down(tree, instr->dest.dest.ssa.bit_size);

   /* Inserting a mov may be unnecessary.  However, it's much easier to
    * simply let copy propagation clean this up than to try to go through
    * and rewrite swizzles ourselves.
    */
   nir_alu_instr *mov = nir_alu_instr_create(mem_ctx, nir_op_imov);
   mov->dest.write_mask = instr->dest.write_mask;
   nir_ssa_dest_init(&mov->instr, &mov->dest.dest,
                     instr->dest.dest.ssa.num_components,
                     instr->dest.dest.ssa.bit_size, NULL);

   mov->src[0] = construct_value(replace,
                                 instr->dest.dest.ssa.num_components, tree,
                                 &state, &instr->instr, mem_ctx);
   nir_instr_insert_before(&instr->instr, &mov->instr);

   nir_ssa_def_rewrite_uses(&instr->dest.dest.ssa,
                            nir_src_for_ssa(&mov->dest.dest.ssa));

   /* We know this one has no more uses because we just rewrote them all,
    * so we can remove it.  The rest of the matched expression, however, we
    * don't know so much about.  We'll just let dead code clean them up.
    */
   nir_instr_remove(&instr->instr);

   ralloc_free(bitsize_ctx);

   return mov;
}