1# Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Tests for Adagrad.""" 16 17from __future__ import absolute_import 18from __future__ import division 19from __future__ import print_function 20 21import numpy as np 22 23from tensorflow.compiler.tests.xla_test import XLATestCase 24from tensorflow.python.framework import constant_op 25from tensorflow.python.ops import resource_variable_ops 26from tensorflow.python.ops import variables 27from tensorflow.python.platform import test 28from tensorflow.python.training import adagrad 29 30 31class AdagradOptimizerTest(XLATestCase): 32 33 def testBasic(self): 34 for dtype in self.float_types: 35 with self.test_session(), self.test_scope(): 36 var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype) 37 var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype) 38 grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) 39 grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) 40 ada_opt = adagrad.AdagradOptimizer(3.0, initial_accumulator_value=0.1) 41 ada_update = ada_opt.apply_gradients( 42 zip([grads0, grads1], [var0, var1])) 43 variables.global_variables_initializer().run() 44 # Fetch params to validate initial values 45 self.assertAllClose([1.0, 2.0], var0.eval()) 46 self.assertAllClose([3.0, 4.0], var1.eval()) 47 # Run 3 steps of adagrad 48 for _ in range(3): 49 ada_update.run() 50 # Validate updated params 51 self.assertAllCloseAccordingToType( 52 np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval(), 53 float_rtol=1e-5) 54 self.assertAllCloseAccordingToType( 55 np.array([2.715679168701172, 3.715679168701172]), var1.eval(), 56 float_rtol=1e-5) 57 58 def testTensorLearningRate(self): 59 for dtype in self.float_types: 60 with self.test_session(), self.test_scope(): 61 var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype) 62 var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype) 63 grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) 64 grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) 65 ada_opt = adagrad.AdagradOptimizer( 66 constant_op.constant(3.0), initial_accumulator_value=0.1) 67 ada_update = ada_opt.apply_gradients( 68 zip([grads0, grads1], [var0, var1])) 69 variables.global_variables_initializer().run() 70 # Fetch params to validate initial values 71 self.assertAllClose([1.0, 2.0], var0.eval()) 72 self.assertAllClose([3.0, 4.0], var1.eval()) 73 # Run 3 steps of adagrad 74 for _ in range(3): 75 ada_update.run() 76 # Validate updated params 77 self.assertAllCloseAccordingToType( 78 np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval(), 79 float_rtol=1e-5) 80 self.assertAllCloseAccordingToType( 81 np.array([2.715679168701172, 3.715679168701172]), var1.eval(), 82 float_rtol=1e-5) 83 84 def testSharing(self): 85 for dtype in self.float_types: 86 with self.test_session(), self.test_scope(): 87 var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype) 88 var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype) 89 grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) 90 grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) 91 ada_opt = adagrad.AdagradOptimizer(3.0) 92 # Apply the optimizer twice. Both applications will use 93 # the same accums. 94 ada_update1 = ada_opt.apply_gradients( 95 zip([grads0, grads1], [var0, var1])) 96 ada_update2 = ada_opt.apply_gradients( 97 zip([grads0, grads1], [var0, var1])) 98 self.assertEqual(["accumulator"], ada_opt.get_slot_names()) 99 slot0 = ada_opt.get_slot(var0, "accumulator") 100 self.assertEquals(slot0.get_shape(), var0.get_shape()) 101 slot1 = ada_opt.get_slot(var1, "accumulator") 102 self.assertEquals(slot1.get_shape(), var1.get_shape()) 103 variables.global_variables_initializer().run() 104 105 # Fetch params to validate initial values. 106 self.assertAllClose([1.0, 2.0], var0.eval()) 107 self.assertAllClose([3.0, 4.0], var1.eval()) 108 # Mix the first and the second adagrad for 3 steps. 109 ada_update1.run() 110 ada_update2.run() 111 ada_update1.run() 112 # Validate updated params (the same as with only 1 Adagrad). 113 self.assertAllCloseAccordingToType( 114 np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval(), 115 float_rtol=1e-5) 116 self.assertAllCloseAccordingToType( 117 np.array([2.715679168701172, 3.715679168701172]), var1.eval(), 118 float_rtol=1e-5) 119 120 121if __name__ == "__main__": 122 test.main() 123