1# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""Tests for Adagrad."""
16
17from __future__ import absolute_import
18from __future__ import division
19from __future__ import print_function
20
21import numpy as np
22
23from tensorflow.compiler.tests.xla_test import XLATestCase
24from tensorflow.python.framework import constant_op
25from tensorflow.python.ops import resource_variable_ops
26from tensorflow.python.ops import variables
27from tensorflow.python.platform import test
28from tensorflow.python.training import adagrad
29
30
31class AdagradOptimizerTest(XLATestCase):
32
33  def testBasic(self):
34    for dtype in self.float_types:
35      with self.test_session(), self.test_scope():
36        var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype)
37        var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype)
38        grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
39        grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
40        ada_opt = adagrad.AdagradOptimizer(3.0, initial_accumulator_value=0.1)
41        ada_update = ada_opt.apply_gradients(
42            zip([grads0, grads1], [var0, var1]))
43        variables.global_variables_initializer().run()
44        # Fetch params to validate initial values
45        self.assertAllClose([1.0, 2.0], var0.eval())
46        self.assertAllClose([3.0, 4.0], var1.eval())
47        # Run 3 steps of adagrad
48        for _ in range(3):
49          ada_update.run()
50        # Validate updated params
51        self.assertAllCloseAccordingToType(
52            np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval(),
53            float_rtol=1e-5)
54        self.assertAllCloseAccordingToType(
55            np.array([2.715679168701172, 3.715679168701172]), var1.eval(),
56            float_rtol=1e-5)
57
58  def testTensorLearningRate(self):
59    for dtype in self.float_types:
60      with self.test_session(), self.test_scope():
61        var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype)
62        var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype)
63        grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
64        grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
65        ada_opt = adagrad.AdagradOptimizer(
66            constant_op.constant(3.0), initial_accumulator_value=0.1)
67        ada_update = ada_opt.apply_gradients(
68            zip([grads0, grads1], [var0, var1]))
69        variables.global_variables_initializer().run()
70        # Fetch params to validate initial values
71        self.assertAllClose([1.0, 2.0], var0.eval())
72        self.assertAllClose([3.0, 4.0], var1.eval())
73        # Run 3 steps of adagrad
74        for _ in range(3):
75          ada_update.run()
76        # Validate updated params
77        self.assertAllCloseAccordingToType(
78            np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval(),
79            float_rtol=1e-5)
80        self.assertAllCloseAccordingToType(
81            np.array([2.715679168701172, 3.715679168701172]), var1.eval(),
82            float_rtol=1e-5)
83
84  def testSharing(self):
85    for dtype in self.float_types:
86      with self.test_session(), self.test_scope():
87        var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype)
88        var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype)
89        grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
90        grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
91        ada_opt = adagrad.AdagradOptimizer(3.0)
92        # Apply the optimizer twice.  Both applications will use
93        # the same accums.
94        ada_update1 = ada_opt.apply_gradients(
95            zip([grads0, grads1], [var0, var1]))
96        ada_update2 = ada_opt.apply_gradients(
97            zip([grads0, grads1], [var0, var1]))
98        self.assertEqual(["accumulator"], ada_opt.get_slot_names())
99        slot0 = ada_opt.get_slot(var0, "accumulator")
100        self.assertEquals(slot0.get_shape(), var0.get_shape())
101        slot1 = ada_opt.get_slot(var1, "accumulator")
102        self.assertEquals(slot1.get_shape(), var1.get_shape())
103        variables.global_variables_initializer().run()
104
105        # Fetch params to validate initial values.
106        self.assertAllClose([1.0, 2.0], var0.eval())
107        self.assertAllClose([3.0, 4.0], var1.eval())
108        # Mix the first and the second adagrad for 3 steps.
109        ada_update1.run()
110        ada_update2.run()
111        ada_update1.run()
112        # Validate updated params (the same as with only 1 Adagrad).
113        self.assertAllCloseAccordingToType(
114            np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval(),
115            float_rtol=1e-5)
116        self.assertAllCloseAccordingToType(
117            np.array([2.715679168701172, 3.715679168701172]), var1.eval(),
118            float_rtol=1e-5)
119
120
121if __name__ == "__main__":
122  test.main()
123