nn_grad_test.cc revision fe8406149feec453250905965a14285465cd2063
1/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. 2 3Licensed under the Apache License, Version 2.0 (the "License"); 4you may not use this file except in compliance with the License. 5You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9Unless required by applicable law or agreed to in writing, software 10distributed under the License is distributed on an "AS IS" BASIS, 11WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12See the License for the specific language governing permissions and 13limitations under the License. 14==============================================================================*/ 15 16#include "tensorflow/cc/framework/grad_op_registry.h" 17#include "tensorflow/cc/framework/gradient_checker.h" 18#include "tensorflow/cc/framework/testutil.h" 19#include "tensorflow/cc/gradients/grad_testutil.h" 20#include "tensorflow/cc/ops/standard_ops.h" 21#include "tensorflow/core/framework/tensor_testutil.h" 22#include "tensorflow/core/lib/core/status_test_util.h" 23#include "tensorflow/core/lib/random/random.h" 24 25namespace tensorflow { 26using namespace ops; // NOLINT(build/namespaces) 27 28namespace { 29 30class NNGradTest : public ::testing::Test { 31 protected: 32 NNGradTest() : scope_(Scope::NewRootScope()) {} 33 34 void RunTest(const Output& x, const TensorShape& x_shape, const Output& y, 35 const TensorShape& y_shape) { 36 float max_error; 37 TF_ASSERT_OK((ComputeGradientError<float, float, float>( 38 scope_, {x}, {x_shape}, {y}, {y_shape}, &max_error))); 39 EXPECT_LT(max_error, 1e-3); 40 } 41 42 void RunTest(const Output& x, const Tensor& x_init_value, const Output& y, 43 const TensorShape& y_shape) { 44 float max_error; 45 TF_ASSERT_OK((ComputeGradientError<float, float, float>( 46 scope_, x, x_init_value, y, y_shape, &max_error))); 47 EXPECT_LT(max_error, 1e-3); 48 } 49 50 void RunTest(const OutputList& xs, const std::vector<TensorShape>& x_shapes, 51 const OutputList& ys, const std::vector<TensorShape>& y_shapes) { 52 TF_ASSERT_OK(scope_.status()); 53 float max_error; 54 TF_ASSERT_OK((ComputeGradientError<float, float, float>( 55 scope_, xs, x_shapes, ys, y_shapes, &max_error))); 56 EXPECT_LT(max_error, 1e-3); 57 } 58 59 // Sets tensor with random values, ensuring that the max value is largest by 60 // a reasonable amount. 61 // This is an issue for MaxPool and MaxPoolV2, in which perturbations by the 62 // numeric gradient computation in the gradient checker can change the max 63 // value if values are too close together. 64 template <typename T> 65 void SetRandomValuesWithBumpedMax(Tensor* tensor) { 66 auto tensor_flat = tensor->flat<T>(); 67 tensor_flat.setRandom(); 68 int32 max_index = 0; 69 for (size_t i = 1; i < tensor->NumElements(); i++) { 70 if (tensor_flat(i) > tensor_flat(max_index)) { 71 max_index = i; 72 } 73 } 74 tensor_flat(max_index) += 1e-2; 75 } 76 77 Scope scope_; 78}; 79 80TEST_F(NNGradTest, SoftmaxGrad) { 81 TensorShape shape({32, 10}); 82 auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape)); 83 auto y = Softmax(scope_, x); 84 RunTest(x, shape, y, shape); 85} 86 87TEST_F(NNGradTest, LogSoftmaxGrad) { 88 TensorShape shape({5, 3}); 89 auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape)); 90 auto y = LogSoftmax(scope_, x); 91 // Avoid numerical instability when computing finite differences. 92 Tensor x_init_value = 93 test::AsTensor<float>({-0.9f, -0.7f, -0.5f, -0.3f, -0.1f, 0.1f, 0.3f, 94 0.5f, 0.7f, 0.8f, -0.1f, 0.1f, 0.1f, 0.1f, 1.2f}, 95 {5, 3}); 96 RunTest(x, x_init_value, y, shape); 97} 98 99TEST_F(NNGradTest, ReluGrad) { 100 TensorShape shape({5, 2}); 101 auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape)); 102 auto y = Relu(scope_, x); 103 // Avoid input values where ReLU gradient is not well defined (around zero). 104 Tensor x_init_value = test::AsTensor<float>( 105 {-0.9f, -0.7f, -0.5f, -0.3f, -0.1f, 0.1f, 0.3f, 0.5f, 0.7f, 0.9f}, 106 {5, 2}); 107 RunTest(x, x_init_value, y, shape); 108} 109 110TEST_F(NNGradTest, Relu6Grad) { 111 TensorShape shape({5, 2}); 112 auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape)); 113 auto y = Relu6(scope_, x); 114 // Avoid input values where ReLU gradient is not well defined (around zero 115 // and six). 116 Tensor x_init_value = test::AsTensor<float>( 117 {-0.9f, -0.7f, -0.5f, -0.3f, -0.1f, 6.1f, 6.3f, 6.5f, 6.7f, 6.9f}, 118 {5, 2}); 119 RunTest(x, x_init_value, y, shape); 120} 121 122TEST_F(NNGradTest, EluGrad) { 123 TensorShape shape({5, 2}); 124 auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape)); 125 auto y = Elu(scope_, x); 126 Tensor x_init_value = test::AsTensor<float>( 127 {-0.9f, -0.7f, -0.5f, -0.3f, -0.1f, 0.1f, 0.3f, 0.5f, 0.7f, 0.9f}, 128 {5, 2}); 129 RunTest(x, x_init_value, y, shape); 130} 131 132TEST_F(NNGradTest, SeluGrad) { 133 TensorShape shape({5, 2}); 134 auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape)); 135 auto y = Selu(scope_, x); 136 Tensor x_init_value = test::AsTensor<float>( 137 {-0.9f, -0.7f, -0.5f, -0.3f, -0.1f, 0.1f, 0.3f, 0.5f, 0.7f, 0.9f}, 138 {5, 2}); 139 RunTest(x, x_init_value, y, shape); 140} 141 142TEST_F(NNGradTest, L2LossGrad) { 143 TensorShape x_shape({5, 2}); 144 TensorShape y_shape({1}); 145 auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(x_shape)); 146 auto y = L2Loss(scope_, x); 147 RunTest(x, x_shape, y, y_shape); 148} 149 150TEST_F(NNGradTest, BiasAddGradHelper) { 151 TensorShape shape({4, 5}); 152 TensorShape bias_shape({5}); 153 auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape)); 154 auto bias = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(bias_shape)); 155 auto y = BiasAdd(scope_, x, bias); 156 RunTest({x, bias}, {shape, bias_shape}, {y}, {shape}); 157} 158 159TEST_F(NNGradTest, Conv2DGrad) { 160 TensorShape shape({1, 2, 2, 1}); 161 auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape)); 162 Tensor filter = test::AsTensor<float>({0.5f}, {1, 1, 1, 1}); 163 const std::vector<int> strides{1, 1, 1, 1}; 164 auto y = Conv2D(scope_, x, filter, strides, "SAME"); 165 RunTest(x, shape, y, shape); 166} 167 168TEST_F(NNGradTest, MaxPoolGradHelper) { 169 TensorShape x_shape({1, 2, 2, 1}); 170 TensorShape y_shape({1, 1, 1, 1}); 171 auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(x_shape)); 172 // Setup window and strides so that we only do one MaxPool. 173 const std::vector<int> ksize{1, 2, 2, 1}; 174 const std::vector<int> strides{1, 2, 2, 1}; 175 auto y = MaxPool(scope_, x, ksize, strides, "VALID"); 176 Tensor x_init_value = Tensor(DT_FLOAT, x_shape); 177 SetRandomValuesWithBumpedMax<float>(&x_init_value); 178 RunTest(x, x_init_value, y, y_shape); 179} 180 181TEST_F(NNGradTest, MaxPoolGradV2Helper) { 182 TensorShape x_shape({1, 2, 2, 1}); 183 TensorShape y_shape({1, 1, 1, 1}); 184 auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(x_shape)); 185 // Setup window and strides so that we only do one MaxPool. 186 Tensor ksize = test::AsTensor<int>({1, 2, 2, 1}, {4}); 187 Tensor strides = test::AsTensor<int>({1, 2, 2, 1}, {4}); 188 auto y = MaxPoolV2(scope_, x, ksize, strides, "VALID"); 189 Tensor x_init_value = Tensor(DT_FLOAT, x_shape); 190 SetRandomValuesWithBumpedMax<float>(&x_init_value); 191 RunTest(x, x_init_value, y, y_shape); 192} 193 194TEST_F(NNGradTest, LRN){ 195 TensorShape x_shape({1, 1, 2, 1}); 196 auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(x_shape)); 197 auto y = LRN(scope_, x); 198 RunTest(x, x_shape, y, x_shape); 199} 200 201} // namespace 202} // namespace tensorflow 203