1# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""Functional tests for BiasAdd."""
16
17from __future__ import absolute_import
18from __future__ import division
19from __future__ import print_function
20
21import numpy as np
22
23from tensorflow.python.framework import constant_op
24from tensorflow.python.framework import dtypes
25from tensorflow.python.ops import array_ops
26from tensorflow.python.ops import gradient_checker
27from tensorflow.python.ops import gradients_impl
28from tensorflow.python.ops import nn_ops
29import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
30from tensorflow.python.platform import test
31
32
33def GetTestConfigs():
34  """Get all the valid tests configs to run.
35
36  Returns:
37    all the valid test configs as tuples of data_format and use_gpu.
38  """
39  test_configs = [("NHWC", False), ("NHWC", True)]
40  if test.is_gpu_available(cuda_only=True):
41    # "NCHW" format is currently only supported on CUDA.
42    test_configs += [("NCHW", True)]
43  return test_configs
44
45
46class BiasAddTest(test.TestCase):
47
48  def _npBias(self, inputs, bias):
49    assert len(bias.shape) == 1
50    print(inputs.shape)
51    print(bias.shape)
52    assert inputs.shape[-1] == bias.shape[0]
53    return inputs + bias.reshape(([1] * (len(inputs.shape) - 1)) +
54                                 [bias.shape[0]])
55
56  def testNpBias(self):
57    self.assertAllClose(
58        np.array([[11, 22, 33], [41, 52, 63]]),
59        self._npBias(
60            np.array([[10, 20, 30], [40, 50, 60]]), np.array([1, 2, 3])))
61
62  def _testBias(self, np_inputs, np_bias, use_gpu=False):
63    np_val = self._npBias(np_inputs, np_bias)
64    with self.test_session(use_gpu=use_gpu):
65      tf_val = nn_ops.bias_add(np_inputs, np_bias).eval()
66    self.assertAllCloseAccordingToType(np_val, tf_val)
67
68  def _AtLeast3d(self, np_value):
69    # fill the input value to at least 3-dimension
70    if np_value.ndim < 3:
71      return np.reshape(np_value, (1,) * (3 - np_value.ndim) + np_value.shape)
72    return np_value
73
74  def _NHWCToNCHW(self, np_value):
75    # fill the input value to at least 3-dimension
76    np_value = self._AtLeast3d(np_value)
77    # move the last dimension to third-to-last
78    np_dim = list(range(np_value.ndim))
79    np_dim_new = list(np_dim[0:-3]) + list(np_dim[-1:]) + list(np_dim[-3:-1])
80    return np.transpose(np_value, np_dim_new)
81
82  def _NCHWToNHWC(self, np_value):
83    assert len(np_value.shape) >= 3
84    np_dim = list(range(np_value.ndim))
85    # move the third-to-last dimension to the last
86    np_dim_new = list(np_dim[0:-3]) + list(np_dim[-2:]) + list(np_dim[-3:-2])
87    return np.transpose(np_value, np_dim_new)
88
89  def _testBiasNCHW(self, np_inputs, np_bias, use_gpu):
90    np_val = self._npBias(np_inputs, np_bias)
91    np_inputs = self._NHWCToNCHW(np_inputs)
92    with self.test_session(use_gpu=use_gpu):
93      tf_val = nn_ops.bias_add(np_inputs, np_bias, data_format="NCHW").eval()
94    tf_val = self._NCHWToNHWC(tf_val)
95    self.assertAllCloseAccordingToType(self._AtLeast3d(np_val), tf_val)
96
97  def _testAll(self, np_inputs, np_bias):
98    self._testBias(np_inputs, np_bias, use_gpu=False)
99    if np_inputs.dtype in [np.float16, np.float32, np.float64]:
100      self._testBias(np_inputs, np_bias, use_gpu=True)
101      if test.is_gpu_available(cuda_only=True):
102        self._testBiasNCHW(np_inputs, np_bias, use_gpu=True)
103
104  def testInputDims(self):
105    with self.assertRaises(ValueError):
106      nn_ops.bias_add([1, 2], [1])
107
108  def testBiasVec(self):
109    with self.assertRaises(ValueError):
110      nn_ops.bias_add(
111          array_ops.reshape(
112              [1, 2], shape=[1, 2]),
113          array_ops.reshape(
114              [1, 2], shape=[1, 2]))
115
116  def testBiasInputsMatch(self):
117    with self.assertRaises(ValueError):
118      nn_ops.bias_add(
119          array_ops.reshape(
120              [1, 2], shape=[1, 2]),
121          array_ops.reshape(
122              [1], shape=[1]))
123
124  def testIntTypes(self):
125    for t in [np.int8, np.int16, np.int32, np.int64]:
126      self._testAll(
127          np.array([[10, 20, 30], [40, 50, 60]]).astype(t),
128          np.array([1, 2, 3]).astype(t))
129
130  def testFloatTypes(self):
131    for t in [np.float16, np.float32, np.float64]:
132      self._testAll(
133          np.random.rand(4, 3, 3).astype(t), np.random.rand(3).astype(t))
134
135  def _testGradient(self, np_input, bias, dtype, data_format, use_gpu):
136    with self.test_session(use_gpu=use_gpu):
137      if data_format == "NCHW":
138        np_input = self._NHWCToNCHW(np_input)
139      input_tensor = constant_op.constant(
140          np_input, shape=np_input.shape, dtype=dtype)
141      bias_tensor = constant_op.constant(bias, shape=bias.shape, dtype=dtype)
142      output_tensor = nn_ops.bias_add(
143          input_tensor, bias_tensor, data_format=data_format)
144      tensor_jacob_t, tensor_jacob_n = gradient_checker.compute_gradient(
145          input_tensor, np_input.shape, output_tensor, np_input.shape)
146      bias_jacob_t, bias_jacob_n = gradient_checker.compute_gradient(
147          bias_tensor, bias.shape, output_tensor, np_input.shape)
148
149      # Test gradient of BiasAddGrad
150      bias_add_grad = gradients_impl.gradients(
151          nn_ops.l2_loss(output_tensor), bias_tensor)[0]
152      grad_jacob_t, grad_jacob_n = gradient_checker.compute_gradient(
153          output_tensor, np_input.shape, bias_add_grad, bias.shape)
154
155      if dtype == np.float16:
156        # Compare fp16 theoretical gradients to fp32 numerical gradients,
157        # since fp16 numerical gradients are too imprecise unless great
158        # care is taken with choosing the inputs and the delta. This is
159        # a weaker check (in particular, it does not test the op itself,
160        # only its gradient), but it's much better than nothing.
161        input_tensor = constant_op.constant(
162            np_input, shape=np_input.shape, dtype=np.float32)
163        bias_tensor = constant_op.constant(
164            bias, shape=bias.shape, dtype=np.float32)
165        output_tensor = nn_ops.bias_add(
166            input_tensor, bias_tensor, data_format=data_format)
167        _, tensor_jacob_n = gradient_checker.compute_gradient(input_tensor,
168                                                              np_input.shape,
169                                                              output_tensor,
170                                                              np_input.shape)
171        _, bias_jacob_n = gradient_checker.compute_gradient(bias_tensor,
172                                                            bias.shape,
173                                                            output_tensor,
174                                                            np_input.shape)
175
176        bias_add_grad = gradients_impl.gradients(
177            nn_ops.l2_loss(output_tensor), bias_tensor)[0]
178        _, grad_jacob_n = gradient_checker.compute_gradient(output_tensor,
179                                                            np_input.shape,
180                                                            bias_add_grad,
181                                                            bias.shape)
182
183      threshold = 2e-3
184      if dtype == dtypes.float64:
185        threshold = 1e-10
186      self.assertAllClose(tensor_jacob_t, tensor_jacob_n, threshold, threshold)
187      self.assertAllClose(bias_jacob_t, bias_jacob_n, threshold, threshold)
188      self.assertAllClose(grad_jacob_t, grad_jacob_n, threshold, threshold)
189
190  def testGradientTensor(self):
191    for (data_format, use_gpu) in GetTestConfigs():
192      for dtype in (dtypes.float16, dtypes.float32, dtypes.float64):
193        np_input = np.array(
194            [1.0, 2.0, 3.0, 4.0, 5.0, 6.0],
195            dtype=dtype.as_numpy_dtype).reshape(3, 2)
196        bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype)
197        self._testGradient(np_input, bias, dtype, data_format, use_gpu)
198
199  def testGradientTensor4D(self):
200    for (data_format, use_gpu) in GetTestConfigs():
201      for dtype in (dtypes.float16, dtypes.float32, dtypes.float64):
202        np_input = np.arange(
203            1.0, 49.0, dtype=dtype.as_numpy_dtype).reshape(
204                [2, 3, 4, 2]).astype(np.float32)
205        bias = np.array([1.3, 2.4], dtype=dtype.as_numpy_dtype)
206        self._testGradient(np_input, bias, dtype, data_format, use_gpu)
207
208  def testEmpty(self):
209    np.random.seed(7)
210    for shape in (0, 0), (2, 0), (0, 2), (4, 3, 0), (4, 0, 3), (0, 4, 3):
211      self._testAll(np.random.randn(*shape), np.random.randn(shape[-1]))
212
213  def testEmptyGradient(self):
214    for data_format, use_gpu in GetTestConfigs():
215      for shape in (0, 0), (2, 0), (0, 2), (4, 3, 0), (4, 0, 3), (0, 4, 3):
216        self._testGradient(
217            np.random.randn(*shape),
218            np.random.randn(shape[-1]), dtypes.float64, data_format, use_gpu)
219
220
221if __name__ == "__main__":
222  test.main()
223