nn_grad_test.cc revision 7766611a1939842316997214bc911a80fcee4697 
1/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
2
3Licensed under the Apache License, Version 2.0 (the "License");
4you may not use this file except in compliance with the License.
5You may obtain a copy of the License at
6
7    http://www.apache.org/licenses/LICENSE-2.0
8
9Unless required by applicable law or agreed to in writing, software
10distributed under the License is distributed on an "AS IS" BASIS,
11WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12See the License for the specific language governing permissions and
13limitations under the License.
14==============================================================================*/
15
16#include "tensorflow/cc/framework/grad_op_registry.h"
17#include "tensorflow/cc/framework/gradient_checker.h"
18#include "tensorflow/cc/framework/testutil.h"
19#include "tensorflow/cc/gradients/grad_testutil.h"
20#include "tensorflow/cc/ops/standard_ops.h"
21#include "tensorflow/core/framework/tensor_testutil.h"
22#include "tensorflow/core/lib/core/status_test_util.h"
23#include "tensorflow/core/lib/random/random.h"
24
25namespace tensorflow {
26using namespace ops;  // NOLINT(build/namespaces)
27
28namespace {
29
30class NNGradTest : public ::testing::Test {
31 protected:
32  NNGradTest() : scope_(Scope::NewRootScope()) {}
33
34  void RunTest(const Output& x, const TensorShape& x_shape, const Output& y,
35               const TensorShape& y_shape) {
36    float max_error;
37    TF_ASSERT_OK((ComputeGradientError<float, float, float>(
38        scope_, {x}, {x_shape}, {y}, {y_shape}, &max_error)));
39    EXPECT_LT(max_error, 1e-3);
40  }
41
42  void RunTest(const Output& x, const Tensor& x_init_value, const Output& y,
43               const TensorShape& y_shape) {
44    float max_error;
45    TF_ASSERT_OK((ComputeGradientError<float, float, float>(
46        scope_, x, x_init_value, y, y_shape, &max_error)));
47    EXPECT_LT(max_error, 1e-3);
48  }
49
50  void RunTest(const OutputList& xs, const std::vector<TensorShape>& x_shapes,
51               const OutputList& ys, const std::vector<TensorShape>& y_shapes) {
52    TF_ASSERT_OK(scope_.status());
53    float max_error;
54    TF_ASSERT_OK((ComputeGradientError<float, float, float>(
55        scope_, xs, x_shapes, ys, y_shapes, &max_error)));
56    EXPECT_LT(max_error, 1e-3);
57  }
58
59  // Sets tensor with random values, ensuring that the max value is largest by
60  // a reasonable amount.
61  // This is an issue for MaxPool and MaxPoolV2, in which perturbations by the
62  // numeric gradient computation in the gradient checker can change the max
63  // value if values are too close together.
64  template <typename T>
65  void SetRandomValuesWithBumpedMax(Tensor* tensor) {
66    auto tensor_flat = tensor->flat<T>();
67    tensor_flat.setRandom();
68    int32 max_index = 0;
69    for (size_t i = 1; i < tensor->NumElements(); i++) {
70      if (tensor_flat(i) > tensor_flat(max_index)) {
71        max_index = i;
72      }
73    }
74    tensor_flat(max_index) += 1e-2;
75  }
76
77  Scope scope_;
78};
79
80TEST_F(NNGradTest, SoftmaxGrad) {
81  TensorShape shape({32, 10});
82  auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape));
83  auto y = Softmax(scope_, x);
84  RunTest(x, shape, y, shape);
85}
86
87TEST_F(NNGradTest, LogSoftmaxGrad) {
88  TensorShape shape({5, 3});
89  auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape));
90  auto y = LogSoftmax(scope_, x);
91  // Avoid numerical instability when computing finite differences.
92  Tensor x_init_value =
93      test::AsTensor<float>({-0.9f, -0.7f, -0.5f, -0.3f, -0.1f, 0.1f, 0.3f,
94                             0.5f, 0.7f, 0.8f, -0.1f, 0.1f, 0.1f, 0.1f, 1.2f},
95                            {5, 3});
96  RunTest(x, x_init_value, y, shape);
97}
98
99TEST_F(NNGradTest, ReluGrad) {
100  TensorShape shape({5, 2});
101  auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape));
102  auto y = Relu(scope_, x);
103  // Avoid input values where ReLU gradient is not well defined (around zero).
104  Tensor x_init_value = test::AsTensor<float>(
105      {-0.9f, -0.7f, -0.5f, -0.3f, -0.1f, 0.1f, 0.3f, 0.5f, 0.7f, 0.9f},
106      {5, 2});
107  RunTest(x, x_init_value, y, shape);
108}
109
110TEST_F(NNGradTest, Relu6Grad) {
111  TensorShape shape({5, 2});
112  auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape));
113  auto y = Relu6(scope_, x);
114  // Avoid input values where ReLU gradient is not well defined (around zero
115  // and six).
116  Tensor x_init_value = test::AsTensor<float>(
117      {-0.9f, -0.7f, -0.5f, -0.3f, -0.1f, 6.1f, 6.3f, 6.5f, 6.7f, 6.9f},
118      {5, 2});
119  RunTest(x, x_init_value, y, shape);
120}
121
122TEST_F(NNGradTest, EluGrad) {
123  TensorShape shape({5, 2});
124  auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape));
125  auto y = Elu(scope_, x);
126  Tensor x_init_value = test::AsTensor<float>(
127      {-0.9f, -0.7f, -0.5f, -0.3f, -0.1f, 0.1f, 0.3f, 0.5f, 0.7f, 0.9f},
128      {5, 2});
129  RunTest(x, x_init_value, y, shape);
130}
131
132TEST_F(NNGradTest, SeluGrad) {
133  TensorShape shape({5, 2});
134  auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape));
135  auto y = Selu(scope_, x);
136  Tensor x_init_value = test::AsTensor<float>(
137      {-0.9f, -0.7f, -0.5f, -0.3f, -0.1f, 0.1f, 0.3f, 0.5f, 0.7f, 0.9f},
138      {5, 2});
139  RunTest(x, x_init_value, y, shape);
140}
141
142TEST_F(NNGradTest, L2LossGrad) {
143  TensorShape x_shape({5, 2});
144  TensorShape y_shape({1});
145  auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(x_shape));
146  auto y = L2Loss(scope_, x);
147  RunTest(x, x_shape, y, y_shape);
148}
149
150TEST_F(NNGradTest, BiasAddGradHelper) {
151  TensorShape shape({4, 5});
152  TensorShape bias_shape({5});
153  auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape));
154  auto bias = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(bias_shape));
155  auto y = BiasAdd(scope_, x, bias);
156  RunTest({x, bias}, {shape, bias_shape}, {y}, {shape});
157}
158
159TEST_F(NNGradTest, Conv2DGrad) {
160  TensorShape shape({1, 2, 2, 1});
161  auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(shape));
162  Tensor filter = test::AsTensor<float>({0.5f}, {1, 1, 1, 1});
163  const std::vector<int> strides{1, 1, 1, 1};
164  auto y = Conv2D(scope_, x, filter, strides, "SAME");
165  RunTest(x, shape, y, shape);
166}
167
168TEST_F(NNGradTest, MaxPoolGradHelper) {
169  TensorShape x_shape({1, 2, 2, 1});
170  TensorShape y_shape({1, 1, 1, 1});
171  auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(x_shape));
172  // Setup window and strides so that we only do one MaxPool.
173  const std::vector<int> ksize{1, 2, 2, 1};
174  const std::vector<int> strides{1, 2, 2, 1};
175  auto y = MaxPool(scope_, x, ksize, strides, "VALID");
176  Tensor x_init_value = Tensor(DT_FLOAT, x_shape);
177  SetRandomValuesWithBumpedMax<float>(&x_init_value);
178  RunTest(x, x_init_value, y, y_shape);
179}
180
181TEST_F(NNGradTest, MaxPoolGradV2Helper) {
182  TensorShape x_shape({1, 2, 2, 1});
183  TensorShape y_shape({1, 1, 1, 1});
184  auto x = Placeholder(scope_, DT_FLOAT, Placeholder::Shape(x_shape));
185  // Setup window and strides so that we only do one MaxPool.
186  Tensor ksize = test::AsTensor<int>({1, 2, 2, 1}, {4});
187  Tensor strides = test::AsTensor<int>({1, 2, 2, 1}, {4});
188  auto y = MaxPoolV2(scope_, x, ksize, strides, "VALID");
189  Tensor x_init_value = Tensor(DT_FLOAT, x_shape);
190  SetRandomValuesWithBumpedMax<float>(&x_init_value);
191  RunTest(x, x_init_value, y, y_shape);
192}
193
194}  // namespace
195}  // namespace tensorflow
196