1# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""Tests for tensorflow.learning.training_ops."""
16
17from __future__ import absolute_import
18from __future__ import division
19from __future__ import print_function
20
21import itertools
22
23import numpy as np
24
25from tensorflow.python.framework import constant_op
26from tensorflow.python.framework import dtypes
27from tensorflow.python.framework.test_util import TensorFlowTestCase
28# Import resource_variable_ops for the variables-to-tensor implicit conversion.
29from tensorflow.python.ops import resource_variable_ops  # pylint: disable=unused-import
30from tensorflow.python.ops import variables
31from tensorflow.python.platform import googletest
32from tensorflow.python.training import training_ops
33
34
35class TrainingOpsTest(TensorFlowTestCase):
36
37  def _toType(self, dtype):
38    if dtype == np.float16:
39      return dtypes.float16
40    elif dtype == np.float32:
41      return dtypes.float32
42    elif dtype == np.float64:
43      return dtypes.float64
44    elif dtype == np.int32:
45      return dtypes.int32
46    elif dtype == np.int64:
47      return dtypes.int64
48    else:
49      assert False, (dtype)
50
51  def _testTypes(self, x, alpha, delta, use_gpu=None):
52    self.setUp()
53    with self.test_session(use_gpu=use_gpu):
54      var = variables.Variable(x)
55      variables.global_variables_initializer().run()
56      self.assertAllCloseAccordingToType(x, var.eval())
57      apply_sgd = training_ops.apply_gradient_descent(var, alpha, delta)
58      out = apply_sgd.eval()
59      self.assertShapeEqual(out, apply_sgd)
60      self.assertAllCloseAccordingToType(x - alpha * delta, out)
61
62  def testApplyGradientDescent(self):
63    for (dtype, use_gpu) in itertools.product(
64        [np.float16, np.float32, np.float64], [False, True]):
65      x = np.arange(100).astype(dtype)
66      alpha = np.array(2.0).astype(dtype)
67      delta = np.arange(100).astype(dtype)
68      self._testTypes(x, alpha, delta, use_gpu)
69
70  def _testTypesForAdagrad(self, x, y, lr, grad, use_gpu=None):
71    self.setUp()
72    with self.test_session(use_gpu=use_gpu):
73      var = variables.Variable(x)
74      accum = variables.Variable(y)
75      variables.global_variables_initializer().run()
76
77      self.assertAllCloseAccordingToType(x, var.eval())
78      apply_adagrad = training_ops.apply_adagrad(var, accum, lr, grad)
79      out = apply_adagrad.eval()
80      self.assertShapeEqual(out, apply_adagrad)
81      self.assertAllCloseAccordingToType(x - lr * grad * (y + grad * grad)**
82                                         (-0.5), out)
83      self.assertAllCloseAccordingToType(y + grad * grad, accum.eval())
84
85  def _testTypesForFtrl(self,
86                        x,
87                        y,
88                        z,
89                        lr,
90                        grad,
91                        use_gpu=None,
92                        l1=0.0,
93                        l2=0.0,
94                        lr_power=-0.5):
95    self.setUp()
96    with self.test_session(use_gpu=use_gpu):
97      var = variables.Variable(x)
98      accum = variables.Variable(y)
99      linear = variables.Variable(z)
100      variables.global_variables_initializer().run()
101
102      self.assertAllCloseAccordingToType(x, var.eval())
103      apply_ftrl = training_ops.apply_ftrl(var, accum, linear, grad, lr, l1, l2,
104                                           lr_power)
105      out = apply_ftrl.eval()
106      self.assertShapeEqual(out, apply_ftrl)
107      accum_update = y + grad * grad
108      linear_update = z + grad - (accum_update**(-lr_power) - y**
109                                  (-lr_power)) / lr * x
110      quadratic = 1.0 / (accum_update**(lr_power) * lr) + 2 * l2
111      expected_out = np.array([(
112          np.sign(linear_update[i]) * l1 - linear_update[i]) / (quadratic[i]) if
113                               np.abs(linear_update[i]) > l1 else 0.0
114                               for i in range(linear_update.size)])
115      self.assertAllCloseAccordingToType(accum_update, accum.eval())
116      if x.dtype == np.float16:
117        # The calculations here really are not very precise in float16.
118        self.assertAllClose(linear_update, linear.eval(), rtol=2e-2, atol=2e-2)
119        self.assertAllClose(expected_out, out, rtol=2e-2, atol=2e-2)
120      elif x.dtype == np.float32:
121        # The calculations here not sufficiently precise in float32.
122        self.assertAllClose(linear_update, linear.eval(), rtol=1e-5, atol=1e-5)
123        self.assertAllClose(expected_out, out, rtol=1e-5, atol=1e-5)
124      else:
125        self.assertAllClose(linear_update, linear.eval())
126        self.assertAllClose(expected_out, out)
127
128  def testApplyAdagrad(self):
129    for (dtype, use_gpu) in itertools.product(
130        [np.float16, np.float32, np.float64], [False, True]):
131      x = np.arange(100).astype(dtype)
132      y = np.arange(1, 101).astype(dtype)
133      lr = np.array(2.0).astype(dtype)
134      grad = np.arange(100).astype(dtype)
135      self._testTypesForAdagrad(x, y, lr, grad, use_gpu)
136
137  def testApplyFtrl(self):
138    for dtype in [np.float16, np.float32, np.float64]:
139      x = np.arange(100).astype(dtype)
140      y = np.arange(1, 101).astype(dtype)
141      z = np.arange(102, 202).astype(dtype)
142      lr = np.array(2.0).astype(dtype)
143      l1 = np.array(3.0).astype(dtype)
144      l2 = np.array(4.0).astype(dtype)
145      grad = np.arange(100).astype(dtype)
146      self._testTypesForFtrl(x, y, z, lr, grad, use_gpu=False, l1=l1, l2=l2)
147
148  def _testTypesForSparseAdagrad(self, x, y, lr, grad, indices):
149    self.setUp()
150    with self.test_session(use_gpu=False):
151      var = variables.Variable(x)
152      accum = variables.Variable(y)
153      variables.global_variables_initializer().run()
154
155      self.assertAllCloseAccordingToType(x, var.eval())
156      sparse_apply_adagrad = training_ops.sparse_apply_adagrad(
157          var, accum, lr, grad,
158          constant_op.constant(indices, self._toType(indices.dtype)))
159      out = sparse_apply_adagrad.eval()
160      self.assertShapeEqual(out, sparse_apply_adagrad)
161
162      for (i, index) in enumerate(indices):
163        self.assertAllCloseAccordingToType(
164            x[index] - lr * grad[i] * (y[index] + grad[i] * grad[i])**(-0.5),
165            var.eval()[index])
166        self.assertAllCloseAccordingToType(y[index] + grad[i] * grad[i],
167                                           accum.eval()[index])
168
169  def _testTypesForSparseFtrl(self,
170                              x,
171                              y,
172                              z,
173                              lr,
174                              grad,
175                              indices,
176                              l1=0.0,
177                              l2=0.0,
178                              lr_power=-0.5):
179    self.setUp()
180    with self.test_session(use_gpu=False):
181      var = variables.Variable(x)
182      accum = variables.Variable(y)
183      linear = variables.Variable(z)
184      variables.global_variables_initializer().run()
185
186      self.assertAllCloseAccordingToType(x, var.eval())
187      sparse_apply_ftrl = training_ops.sparse_apply_ftrl(
188          var,
189          accum,
190          linear,
191          grad,
192          constant_op.constant(indices, self._toType(indices.dtype)),
193          lr,
194          l1,
195          l2,
196          lr_power=lr_power)
197      out = sparse_apply_ftrl.eval()
198      self.assertShapeEqual(out, sparse_apply_ftrl)
199
200      for (i, index) in enumerate(indices):
201        self.assertAllCloseAccordingToType(x[index] - lr * grad[i] *
202                                           (y[index] + grad[i] * grad[i])**
203                                           (lr_power), var.eval()[index])
204        self.assertAllCloseAccordingToType(y[index] + grad[i] * grad[i],
205                                           accum.eval()[index])
206
207  def testSparseApplyAdagrad(self):
208    for (dtype, index_type) in itertools.product(
209        [np.float16, np.float32, np.float64], [np.int32, np.int64]):
210      x_val = [np.arange(10), np.arange(10, 20), np.arange(20, 30)]
211      y_val = [np.arange(1, 11), np.arange(11, 21), np.arange(21, 31)]
212      x = np.array(x_val).astype(dtype)
213      y = np.array(y_val).astype(dtype)
214      lr = np.array(2.0).astype(dtype)
215      grad_val = [np.arange(10), np.arange(10)]
216      grad = np.array(grad_val).astype(dtype)
217      indices = np.array([0, 2]).astype(index_type)
218      self._testTypesForSparseAdagrad(x, y, lr, grad, indices)
219
220  def testSparseApplyAdagradDim1(self):
221    for (dtype, index_type) in itertools.product(
222        [np.float16, np.float32, np.float64], [np.int32, np.int64]):
223      x_val = [[1.0], [2.0], [3.0]]
224      y_val = [[4.0], [5.0], [6.0]]
225      x = np.array(x_val).astype(dtype)
226      y = np.array(y_val).astype(dtype)
227      lr = np.array(2.0).astype(dtype)
228      grad_val = [[1.5], [2.5]]
229      grad = np.array(grad_val).astype(dtype)
230      indices = np.array([0, 2]).astype(index_type)
231      self._testTypesForSparseAdagrad(x, y, lr, grad, indices)
232
233  def testSparseApplyFtrlDim1(self):
234    for (dtype, index_type) in itertools.product(
235        [np.float16, np.float32, np.float64], [np.int32, np.int64]):
236      x_val = [[0.0], [0.0], [0.0]]
237      y_val = [[4.0], [5.0], [6.0]]
238      z_val = [[0.0], [0.0], [0.0]]
239      x = np.array(x_val).astype(dtype)
240      y = np.array(y_val).astype(dtype)
241      z = np.array(z_val).astype(dtype)
242      lr = np.array(2.0).astype(dtype)
243      grad_val = [[1.5], [2.5]]
244      grad = np.array(grad_val).astype(dtype)
245      indices = np.array([0, 2]).astype(index_type)
246      self._testTypesForSparseFtrl(x, y, z, lr, grad, indices)
247
248  def testApplyAdam(self):
249    for dtype, use_gpu in itertools.product(
250        [np.float16, np.float32, np.float64], [False, True]):
251      var = np.arange(100).astype(dtype)
252      m = np.arange(1, 101).astype(dtype)
253      v = np.arange(101, 201).astype(dtype)
254      grad = np.arange(100).astype(dtype)
255      self._testTypesForAdam(var, m, v, grad, use_gpu)
256
257  def _testTypesForAdam(self, var, m, v, grad, use_gpu):
258    self.setUp()
259    with self.test_session(use_gpu=use_gpu):
260      var_t = variables.Variable(var)
261      m_t = variables.Variable(m)
262      v_t = variables.Variable(v)
263
264      t = 1
265      beta1 = np.array(0.9, dtype=var.dtype)
266      beta2 = np.array(0.999, dtype=var.dtype)
267      beta1_power = beta1**t
268      beta2_power = beta2**t
269      lr = np.array(0.001, dtype=var.dtype)
270      epsilon = np.array(1e-8, dtype=var.dtype)
271      beta1_t = constant_op.constant(beta1, self._toType(var.dtype), [])
272      beta2_t = constant_op.constant(beta2, self._toType(var.dtype), [])
273      beta1_power_t = variables.Variable(beta1_power)
274      beta2_power_t = variables.Variable(beta2_power)
275      lr_t = constant_op.constant(lr, self._toType(var.dtype), [])
276      epsilon_t = constant_op.constant(epsilon, self._toType(var.dtype), [])
277      variables.global_variables_initializer().run()
278
279      self.assertAllCloseAccordingToType(var, var_t.eval())
280      new_var, _, _ = self._adamUpdateNumpy(var, grad, t, m, v, lr, beta1,
281                                            beta2, epsilon)
282      apply_adam = training_ops.apply_adam(var_t, m_t, v_t, beta1_power_t,
283                                           beta2_power_t, lr_t, beta1_t,
284                                           beta2_t, epsilon_t, grad)
285      out = apply_adam.eval()
286      self.assertShapeEqual(out, apply_adam)
287      self.assertAllCloseAccordingToType(new_var, out)
288
289  def _adamUpdateNumpy(self, param, g_t, t, m, v, alpha, beta1, beta2, epsilon):
290    alpha_t = alpha * np.sqrt(1 - beta2**t) / (1 - beta1**t)
291
292    m_t = beta1 * m + (1 - beta1) * g_t
293    v_t = beta2 * v + (1 - beta2) * g_t * g_t
294
295    param_t = param - alpha_t * m_t / (np.sqrt(v_t) + epsilon)
296    return param_t, m_t, v_t
297
298
299if __name__ == '__main__':
300  googletest.main()
301