1# Copyright 2015 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""Tests for tensorflow.ops.tensor_array_ops.""" 16 17from __future__ import absolute_import 18from __future__ import division 19from __future__ import print_function 20 21import numpy as np 22 23from tensorflow.core.protobuf import config_pb2 24from tensorflow.python.client import session as session_lib 25from tensorflow.python.eager import backprop 26from tensorflow.python.eager import context 27from tensorflow.python.framework import constant_op 28from tensorflow.python.framework import dtypes 29from tensorflow.python.framework import ops 30from tensorflow.python.framework import tensor_shape 31from tensorflow.python.framework import test_util 32from tensorflow.python.ops import array_ops 33from tensorflow.python.ops import control_flow_ops 34from tensorflow.python.ops import gen_data_flow_ops 35from tensorflow.python.ops import gradients_impl 36from tensorflow.python.ops import init_ops 37from tensorflow.python.ops import math_ops 38from tensorflow.python.ops import tensor_array_grad 39from tensorflow.python.ops import tensor_array_ops 40from tensorflow.python.ops import variable_scope 41from tensorflow.python.ops import variables 42import tensorflow.python.ops.nn_grad # pylint: disable=unused-import 43from tensorflow.python.platform import test 44 45 46def _make_converter(tf_dtype): 47 def _converter(x): 48 if tf_dtype == dtypes.string: 49 # In Python3, np.str is unicode, while we always want bytes 50 return np.asarray(x).astype("|S") 51 x = np.asarray(x).astype(tf_dtype.as_numpy_dtype) 52 if tf_dtype.is_complex: 53 # Add a non-zero imaginary component to x. 54 x -= 1j * x 55 return x 56 return _converter 57 58 59def _make_ta(size, name, dtype=dtypes.float32, infer_shape=False): 60 return tensor_array_ops.TensorArray( 61 dtype=dtype, tensor_array_name=name, size=size, infer_shape=infer_shape) 62 63 64class TensorArrayTest(test.TestCase): 65 66 @classmethod 67 def setUpClass(cls): 68 super(TensorArrayTest, cls).setUpClass() 69 cls._workers, _ = test.create_local_cluster(num_workers=3, num_ps=0) 70 71 @classmethod 72 def tearDownClass(cls): 73 super(TensorArrayTest, cls).tearDownClass() 74 session_lib.Session.reset(cls._workers[0].target) 75 76 @test_util.run_in_graph_and_eager_modes() 77 def testTensorArrayWriteRead(self): 78 with self.test_session(use_gpu=True): 79 ta = tensor_array_ops.TensorArray( 80 dtype=dtypes.float32, 81 tensor_array_name="foo", 82 size=3, 83 infer_shape=False) 84 85 w0 = ta.write(0, [[4.0, 5.0]]) 86 w1 = w0.write(1, [[1.0]]) 87 w2 = w1.write(2, -3.0) 88 89 r0 = w2.read(0) 90 r1 = w2.read(1) 91 r2 = w2.read(2) 92 93 d0, d1, d2 = self.evaluate([r0, r1, r2]) 94 self.assertAllEqual([[4.0, 5.0]], d0) 95 self.assertAllEqual([[1.0]], d1) 96 self.assertAllEqual(-3.0, d2) 97 98 def _testTensorArrayWritePack(self, tf_dtype): 99 with self.test_session(use_gpu=True): 100 ta = tensor_array_ops.TensorArray( 101 dtype=tf_dtype, tensor_array_name="foo", size=3) 102 103 convert = _make_converter(tf_dtype) 104 105 w0 = ta.write(0, convert([[4.0, 5.0]])) 106 w1 = w0.write(1, convert([[6.0, 7.0]])) 107 w2 = w1.write(2, convert([[8.0, 9.0]])) 108 109 c0 = w2.stack() 110 111 c0 = self.evaluate(c0) 112 self.assertAllEqual( 113 convert([[[4.0, 5.0]], [[6.0, 7.0]], [[8.0, 9.0]]]), c0) 114 115 def _testTensorArrayWritePackMaybeLegacy(self): 116 self._testTensorArrayWritePack(dtypes.float32) 117 self._testTensorArrayWritePack(dtypes.float64) 118 self._testTensorArrayWritePack(dtypes.int32) 119 self._testTensorArrayWritePack(dtypes.int64) 120 self._testTensorArrayWritePack(dtypes.complex64) 121 self._testTensorArrayWritePack(dtypes.complex128) 122 self._testTensorArrayWritePack(dtypes.string) 123 124 @test_util.run_in_graph_and_eager_modes() 125 def testTensorArrayWritePack(self): 126 self._testTensorArrayWritePackMaybeLegacy() 127 128 @test_util.run_in_graph_and_eager_modes() 129 def testEmptyTensorArrayPack(self): 130 with self.test_session(use_gpu=True): 131 ta = tensor_array_ops.TensorArray( 132 dtype=dtypes.float32, tensor_array_name="foo", size=3) 133 134 empty_element = np.zeros((0, 1), dtype=np.float32) 135 w0 = ta.write(0, empty_element) 136 w1 = w0.write(1, empty_element) 137 w2 = w1.write(2, empty_element) 138 139 c0 = w2.stack() 140 141 c0 = self.evaluate(c0) 142 self.assertAllEqual([3, 0, 1], c0.shape) 143 144 def _testTensorArrayWriteConcat(self, tf_dtype): 145 with self.test_session(use_gpu=True): 146 ta = tensor_array_ops.TensorArray( 147 dtype=tf_dtype, tensor_array_name="foo", size=3, infer_shape=False) 148 149 convert = _make_converter(tf_dtype) 150 151 w0 = ta.write(0, convert([[4.0, 5.0], [104.0, 105.0], [204.0, 205.0]])) 152 w1 = w0.write(1, convert([[6.0, 7.0], [106.0, 107.0]])) 153 w2 = w1.write(2, convert([[8.0, 9.0]])) 154 155 c0 = w2.concat() 156 157 c0 = self.evaluate(c0) 158 self.assertAllEqual( 159 convert([[4.0, 5.0], [104.0, 105.0], [204.0, 205.0], [6.0, 7.0], 160 [106.0, 107.0], [8.0, 9.0]]), c0) 161 162 @test_util.run_in_graph_and_eager_modes() 163 def testTensorArrayWriteConcat(self): 164 self._testTensorArrayWriteConcat(dtypes.float32) 165 self._testTensorArrayWriteConcat(dtypes.float64) 166 self._testTensorArrayWriteConcat(dtypes.int32) 167 self._testTensorArrayWriteConcat(dtypes.int64) 168 self._testTensorArrayWriteConcat(dtypes.complex64) 169 self._testTensorArrayWriteConcat(dtypes.complex128) 170 self._testTensorArrayWriteConcat(dtypes.string) 171 172 def _testTensorArrayReadOrPackNotAllValuesAvailableFillsZeros(self): 173 with self.test_session(use_gpu=True): 174 ta = tensor_array_ops.TensorArray( 175 dtype=dtypes.float32, 176 tensor_array_name="foo", 177 size=3, 178 element_shape=tensor_shape.TensorShape([1, 2])) 179 self.assertAllEqual([[0.0, 0.0]], self.evaluate(ta.read(0))) 180 self.assertAllEqual([[[0.0, 0.0]], [[4.0, 5.0]], [[0.0, 0.0]]], 181 self.evaluate(ta.write(1, [[4.0, 5.0]]).stack())) 182 self.assertAllEqual([[0.0, 0.0], [4.0, 5.0], [0.0, 0.0]], 183 self.evaluate(ta.write(1, [[4.0, 5.0]]).concat())) 184 185 @test_util.run_in_graph_and_eager_modes() 186 def testTensorArrayReadOrPackNotAllValuesAvailableFillsZeros(self): 187 self._testTensorArrayReadOrPackNotAllValuesAvailableFillsZeros() 188 189 def _testTensorArrayReadOrPackNotAllValuesAvailableInferShapeFillsZeros(self): 190 ta = tensor_array_ops.TensorArray( 191 dtype=dtypes.float32, 192 tensor_array_name="foo", 193 size=3) 194 self.assertAllEqual( 195 [[0.0, 0.0]], self.evaluate(ta.write(1, [[4.0, 5.0]]).read(0))) 196 self.assertAllEqual([[[0.0, 0.0]], [[4.0, 5.0]], [[0.0, 0.0]]], 197 self.evaluate(ta.write(1, [[4.0, 5.0]]).stack())) 198 self.assertAllEqual([[0.0, 0.0], [4.0, 5.0], [0.0, 0.0]], 199 self.evaluate(ta.write(1, [[4.0, 5.0]]).concat())) 200 201 @test_util.run_in_graph_and_eager_modes() 202 def testTensorArrayReadOrPackNotAllValuesAvailableInferShapeFillsZeros(self): 203 self._testTensorArrayReadOrPackNotAllValuesAvailableInferShapeFillsZeros() 204 205 def _testTensorArrayUnpackRead(self, tf_dtype): 206 with self.test_session(use_gpu=True): 207 convert = _make_converter(tf_dtype) 208 209 ta = _make_ta(3, "foo", dtype=tf_dtype) 210 # Unpack a vector into scalars 211 w0 = ta.unstack(convert([1.0, 2.0, 3.0])) 212 r0 = w0.read(0) 213 r1 = w0.read(1) 214 r2 = w0.read(2) 215 216 d0, d1, d2 = self.evaluate([r0, r1, r2]) 217 self.assertAllEqual(convert(1.0), d0) 218 self.assertAllEqual(convert(2.0), d1) 219 self.assertAllEqual(convert(3.0), d2) 220 221 # Unpack a matrix into vectors 222 w1 = ta.unstack(convert([[1.0, 1.1], [2.0, 2.1], [3.0, 3.1]])) 223 r0 = w1.read(0) 224 r1 = w1.read(1) 225 r2 = w1.read(2) 226 227 d0, d1, d2 = self.evaluate([r0, r1, r2]) 228 self.assertAllEqual(convert([1.0, 1.1]), d0) 229 self.assertAllEqual(convert([2.0, 2.1]), d1) 230 self.assertAllEqual(convert([3.0, 3.1]), d2) 231 232 # Try unpacking an empty matrix, which should not cause an error. 233 w2 = ta.unstack(convert([[], [], []])) 234 r0 = w2.read(0) 235 r1 = w2.read(1) 236 r2 = w2.read(2) 237 238 d0, d1, d2 = self.evaluate([r0, r1, r2]) 239 self.assertAllEqual(convert([]), d0) 240 self.assertAllEqual(convert([]), d1) 241 self.assertAllEqual(convert([]), d2) 242 243 def _testTensorArrayUnpackReadMaybeLegacy(self): 244 self._testTensorArrayUnpackRead(dtypes.float32) 245 self._testTensorArrayUnpackRead(dtypes.float64) 246 self._testTensorArrayUnpackRead(dtypes.int32) 247 self._testTensorArrayUnpackRead(dtypes.int64) 248 self._testTensorArrayUnpackRead(dtypes.complex64) 249 self._testTensorArrayUnpackRead(dtypes.complex128) 250 self._testTensorArrayUnpackRead(dtypes.string) 251 252 @test_util.run_in_graph_and_eager_modes() 253 def testTensorArrayUnpackRead(self): 254 self._testTensorArrayUnpackReadMaybeLegacy() 255 256 def _testTensorArraySplitRead(self, tf_dtype): 257 with self.test_session(use_gpu=True): 258 convert = _make_converter(tf_dtype) 259 260 # Split an empty vector 261 ta = _make_ta(3, "foo", dtype=tf_dtype) 262 lengths = constant_op.constant([0, 0, 0]) 263 w0 = ta.split(convert([]), lengths=lengths) 264 r0 = w0.read(0) 265 r1 = w0.read(1) 266 r2 = w0.read(2) 267 268 d0, d1, d2 = self.evaluate([r0, r1, r2]) 269 self.assertAllEqual(convert([]), d0) 270 self.assertAllEqual(convert([]), d1) 271 self.assertAllEqual(convert([]), d2) 272 273 # Split a vector 274 lengths = constant_op.constant([2, 0, 1]) 275 w0 = ta.split(convert([1.0, 2.0, 3.0]), lengths=lengths) 276 r0 = w0.read(0) 277 r1 = w0.read(1) 278 r2 = w0.read(2) 279 280 d0, d1, d2 = self.evaluate([r0, r1, r2]) 281 self.assertAllEqual(convert([1.0, 2.0]), d0) 282 self.assertAllEqual(convert([]), d1) 283 self.assertAllEqual(convert([3.0]), d2) 284 285 # Split a matrix 286 lengths = constant_op.constant([2, 0, 1]) 287 w0 = ta.split( 288 convert([[1.0, 101.0], [2.0, 201.0], [3.0, 301.0]]), lengths=lengths) 289 r0 = w0.read(0) 290 r1 = w0.read(1) 291 r2 = w0.read(2) 292 293 d0, d1, d2 = self.evaluate([r0, r1, r2]) 294 self.assertAllEqual(convert([[1.0, 101.0], [2.0, 201.0]]), d0) 295 self.assertAllEqual(convert([]).reshape(0, 2), d1) 296 self.assertAllEqual(convert([[3.0, 301.0]]), d2) 297 298 @test_util.run_in_graph_and_eager_modes() 299 def testTensorArraySplitRead(self): 300 self._testTensorArraySplitRead(dtypes.float32) 301 self._testTensorArraySplitRead(dtypes.float64) 302 self._testTensorArraySplitRead(dtypes.int32) 303 self._testTensorArraySplitRead(dtypes.int64) 304 self._testTensorArraySplitRead(dtypes.complex64) 305 self._testTensorArraySplitRead(dtypes.complex128) 306 self._testTensorArraySplitRead(dtypes.string) 307 308 def testTensorGradArrayWriteRead(self): 309 with self.test_session(use_gpu=True) as session: 310 ta = tensor_array_ops.TensorArray( 311 dtype=dtypes.float32, 312 tensor_array_name="foo", 313 size=3, 314 infer_shape=False) 315 g_ta = ta.grad("grad") 316 317 w0 = ta.write(0, [[4.0, 5.0]]) 318 w1 = w0.write(1, [[1.0]]) 319 w2 = w1.write(2, -3.0) 320 321 g_w0 = g_ta.write(0, [[5.0, 6.0]]) 322 g_w1 = g_w0.write(1, [[2.0]]) 323 g_w2 = g_w1.write(2, -2.0) 324 325 r0 = w2.read(0) 326 r1 = w2.read(1) 327 r2 = w2.read(2) 328 329 g_r0 = g_w2.read(0) 330 g_r1 = g_w2.read(1) 331 g_r2 = g_w2.read(2) 332 333 d0, d1, d2, g_d0, g_d1, g_d2 = session.run([r0, r1, r2, g_r0, g_r1, g_r2]) 334 self.assertAllEqual([[4.0, 5.0]], d0) 335 self.assertAllEqual([[1.0]], d1) 336 self.assertAllEqual(-3.0, d2) 337 self.assertAllEqual([[5.0, 6.0]], g_d0) 338 self.assertAllEqual([[2.0]], g_d1) 339 self.assertAllEqual(-2.0, g_d2) 340 341 def testTensorGradArrayDynamicWriteRead(self): 342 with self.test_session(use_gpu=True) as session: 343 ta = tensor_array_ops.TensorArray( 344 dtype=dtypes.float32, 345 tensor_array_name="foo", 346 size=0, 347 dynamic_size=True, 348 infer_shape=False) 349 350 w0 = ta.write(0, [[4.0, 5.0]]) 351 w1 = w0.write(1, [[1.0]]) 352 w2 = w1.write(2, -3.0) 353 354 g_ta = w2.grad("grad") # Get gradient array here so we know the shape 355 356 s = w2.size() 357 g_s = g_ta.size() 358 359 g_w0 = g_ta.write(0, [[5.0, 6.0]]) 360 g_w1 = g_w0.write(1, [[2.0]]) 361 g_w2 = g_w1.write(2, -2.0) 362 363 r0 = w2.read(0) 364 r1 = w2.read(1) 365 r2 = w2.read(2) 366 367 g_r0 = g_w2.read(0) 368 g_r1 = g_w2.read(1) 369 g_r2 = g_w2.read(2) 370 371 d0, d1, d2, g_d0, g_d1, g_d2, vs, g_vs = session.run( 372 [r0, r1, r2, g_r0, g_r1, g_r2, s, g_s]) 373 self.assertAllEqual([[4.0, 5.0]], d0) 374 self.assertAllEqual([[1.0]], d1) 375 self.assertAllEqual(-3.0, d2) 376 self.assertAllEqual([[5.0, 6.0]], g_d0) 377 self.assertAllEqual([[2.0]], g_d1) 378 self.assertAllEqual(-2.0, g_d2) 379 self.assertAllEqual(3, vs) 380 self.assertAllEqual(3, g_vs) 381 382 def testTensorGradAccessTwiceReceiveSameObject(self): 383 with self.test_session(use_gpu=True) as session: 384 ta = tensor_array_ops.TensorArray( 385 dtype=dtypes.float32, tensor_array_name="foo", size=3) 386 g_ta_0 = ta.grad("grad") 387 g_ta_1 = ta.grad("grad") 388 389 with ops.control_dependencies([g_ta_0.write(0, [[4.0, 5.0]]).flow]): 390 # Write with one gradient handle, read with another copy of it 391 r1_0 = g_ta_1.read(0) 392 393 t_g_ta_0, t_g_ta_1, d_r1_0 = session.run( 394 [g_ta_0.handle.op, g_ta_1.handle.op, r1_0]) 395 self.assertAllEqual(t_g_ta_0, t_g_ta_1) 396 self.assertAllEqual([[4.0, 5.0]], d_r1_0) 397 398 @test_util.run_in_graph_and_eager_modes() 399 def testTensorArrayWriteWrongIndexOrDataTypeFails(self): 400 with self.test_session(use_gpu=True): 401 ta = _make_ta(3, "foo", dtype=dtypes.float32) 402 in_graph_mode = context.in_graph_mode() 403 # Test writing the wrong datatype 404 if in_graph_mode: 405 with self.assertRaisesOpError( 406 "TensorArray dtype is float but Op is trying to write " 407 "dtype string"): 408 self.evaluate(ta.write(0, "wrong_type_scalar").flow) 409 else: 410 with self.assertRaisesOpError( 411 "TensorArray dtype is float32 but Op is trying to write " 412 "dtype string"): 413 self.evaluate(ta.write(0, "wrong_type_scalar").flow) 414 415 if context.in_graph_mode(): 416 with self.assertRaisesOpError( 417 "Tried to write to index -1 but array is not " 418 "resizeable and size is: 3"): 419 self.evaluate(ta.write(-1, 3.0).flow) 420 else: 421 with self.assertRaisesOpError( 422 r"Writing to negative indices \(index -1\) is not allowed."): 423 self.evaluate(ta.write(-1, 3.0).flow) 424 425 # Test reading from too large an index 426 with self.assertRaisesOpError( 427 "Tried to write to index 3 but array is not " 428 "resizeable and size is: 3"): 429 self.evaluate(ta.write(3, 3.0).flow) 430 431 @test_util.run_in_graph_and_eager_modes() 432 def testTensorArrayReadWrongIndexOrDataTypeFails(self): 433 with self.test_session(use_gpu=True): 434 ta = _make_ta(3, "foo", dtype=dtypes.float32) 435 436 w0 = ta.write(0, [[4.0, 5.0]]) 437 438 # Test reading wrong datatype, which is only possible in graph mode 439 if context.in_graph_mode(): 440 r0_bad = gen_data_flow_ops._tensor_array_read_v3( 441 handle=w0.handle, index=0, dtype=dtypes.float64, flow_in=w0.flow) 442 with self.assertRaisesOpError( 443 "TensorArray dtype is float but Op requested dtype double."): 444 r0_bad.eval() 445 446 # Test reading from a negative index, which is not allowed 447 if context.in_graph_mode(): 448 with self.assertRaisesOpError( 449 r"Tried to read from index -1 but array size is: 3"): 450 self.evaluate(ta.read(-1)) 451 else: 452 with self.assertRaisesOpError( 453 r"Reading from negative indices \(index -1\) is not allowed."): 454 self.evaluate(ta.read(-1)) 455 456 # Test reading from too large an index 457 with self.assertRaisesOpError( 458 "Tried to read from index 3 but array size is: 3"): 459 self.evaluate(ta.read(3)) 460 461 @test_util.run_in_graph_and_eager_modes() 462 def testTensorArrayWriteMultipleFails(self): 463 with self.test_session(use_gpu=True): 464 ta = tensor_array_ops.TensorArray( 465 dtype=dtypes.float32, tensor_array_name="foo", size=3) 466 467 with self.assertRaisesOpError( 468 "Could not write to TensorArray index 2 because " 469 "it has already been written to."): 470 if context.in_graph_mode(): 471 self.evaluate(ta.write(2, 3.0).write(2, 3.0).flow) 472 else: 473 self.evaluate(ta.write(2, 3.0).write(2, 3.0)) 474 475 @test_util.run_in_graph_and_eager_modes() 476 def testTensorArrayConcatIncompatibleShapesFails(self): 477 with self.test_session(use_gpu=True): 478 ta = tensor_array_ops.TensorArray( 479 dtype=dtypes.float32, 480 tensor_array_name="foo", 481 size=3, 482 infer_shape=False) 483 484 w1 = ta.write(0, 3.0) 485 w2 = w1.write(1, 4.0) 486 w3 = w2.write(2, [3.0]) 487 488 with self.assertRaisesOpError( 489 "Concat saw a scalar shape at index 0 but requires at least vectors"): 490 self.evaluate(w3.concat()) 491 492 ta = tensor_array_ops.TensorArray( 493 dtype=dtypes.float32, 494 tensor_array_name="foo", 495 size=3, 496 infer_shape=False) 497 498 w1 = ta.write(0, [3.0]) 499 w2 = w1.write(1, [4.0]) 500 w3 = w2.write(2, [[3.0]]) 501 502 # The eager-mode implementation just passes up array_op.concat's error 503 # message. 504 if context.in_graph_mode(): 505 with self.assertRaisesOpError( 506 r"TensorArray has inconsistent shapes. Index 0 has " 507 r"\(excepting dimension 0\) shape: \[\] but index 2 has " 508 r"\(excepting dimension 0\) shape: \[1\]"): 509 self.evaluate(w3.concat()) 510 else: 511 with self.assertRaisesOpError( 512 r".*Ranks of all input tensors should match: shape\[0\] " 513 r"= \[1\] vs\. shape\[2\] = \[1,1\].*"): 514 self.evaluate(w3.concat()) 515 516 @test_util.run_in_graph_and_eager_modes() 517 def testTensorArraySplitIncompatibleShapesFails(self): 518 with self.test_session(use_gpu=True): 519 in_graph_mode = context.in_graph_mode() 520 ta = _make_ta(3, "foo") 521 with self.assertRaisesOpError( 522 r"Expected lengths to be a vector, received shape: \[\]"): 523 if in_graph_mode: 524 lengths = array_ops.placeholder(dtypes.int64) 525 ta.split([1.0, 2.0, 3.0], lengths).flow.eval(feed_dict={lengths: 1}) 526 else: 527 self.evaluate(ta.split([1.0, 2.0, 3.0], 1)) 528 529 with self.assertRaisesOpError( 530 r"Expected sum of lengths to be equal to values.shape\[0\], " 531 r"but sum of lengths is 1 and value's shape is: \[3\]"): 532 if in_graph_mode: 533 self.evaluate(ta.split([1.0, 2.0, 3.0], [1]).flow) 534 else: 535 self.evaluate(ta.split([1.0, 2.0, 3.0], [1])) 536 537 ta = _make_ta(1, "baz") 538 with self.assertRaisesOpError( 539 r"Expected value to be at least a vector, but received shape: \[\]"): 540 if in_graph_mode: 541 self.evaluate(ta.split(1.0, [1]).flow) 542 else: 543 self.evaluate(ta.split(1.0, [1])) 544 545 ta = _make_ta(2, "buz") 546 with self.assertRaisesOpError( 547 r"TensorArray's size is not equal to the size of lengths " 548 r"\(2 vs. 1\), and the TensorArray is not marked as " 549 r"dynamically resizeable"): 550 if in_graph_mode: 551 self.evaluate(ta.split([1.0], [1]).flow) 552 else: 553 self.evaluate(ta.split([1.0], [1])) 554 555 def _testTensorArrayWriteGradientAddMultipleAdds(self, dtype): 556 with self.test_session(use_gpu=True): 557 ta = tensor_array_ops.TensorArray( 558 dtype=dtype, tensor_array_name="foo", size=3, infer_shape=False) 559 ta_grad = ta.grad("grad") 560 561 c = lambda x: np.asarray(x, dtype=dtype.as_numpy_dtype) 562 563 w0 = ta.write(2, c(3.0)) 564 w1 = w0.write(2, c(4.0)) 565 566 w0_grad = ta_grad.write(2, c(3.0)) 567 w1_grad = w0_grad.write(2, c(4.0)) 568 w2_grad = w1_grad.write(2, c(5.0)) 569 570 # Assert that aggregation works correctly 571 self.assertAllEqual(c(12.00), w2_grad.read(2).eval()) 572 573 # Assert that if multiple_writes_aggregate is not enabled, 574 # multiple writes raise an exception. 575 with self.assertRaisesOpError( 576 r"TensorArray foo_.*: Could not write to TensorArray index 2 because " 577 r"it has already been written to."): 578 w1.flow.eval() 579 580 # Using differing shapes causes an exception 581 wb0_grad = ta_grad.write(1, c(1.0)) 582 wb1_grad = wb0_grad.write(1, c([1.0])) 583 584 with self.assertRaisesOpError( 585 r"Could not aggregate to TensorArray index 1 because the " 586 r"existing shape is \[\] but the new input shape is \[1\]"): 587 wb1_grad.flow.eval() 588 589 def testTensorArrayWriteGradientAddMultipleAdds(self): 590 for dtype in (dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float64, 591 dtypes.complex64, dtypes.complex128): 592 self._testTensorArrayWriteGradientAddMultipleAdds(dtype) 593 594 @test_util.run_in_graph_and_eager_modes() 595 def testMultiTensorArray(self): 596 with self.test_session(use_gpu=True): 597 h1 = tensor_array_ops.TensorArray( 598 size=1, dtype=dtypes.float32, tensor_array_name="foo") 599 w1 = h1.write(0, 4.0) 600 r1 = w1.read(0) 601 602 h2 = tensor_array_ops.TensorArray( 603 size=1, dtype=dtypes.float32, tensor_array_name="bar") 604 605 w2 = h2.write(0, 5.0) 606 r2 = w2.read(0) 607 r = r1 + r2 608 val = self.evaluate(r) 609 self.assertAllClose(9.0, val) 610 611 def _testTensorArrayGradientWriteReadType(self, dtype): 612 with self.test_session(use_gpu=True) as session: 613 ta = tensor_array_ops.TensorArray( 614 dtype=dtypes.as_dtype(dtype), 615 tensor_array_name="foo", 616 size=3, 617 infer_shape=False) 618 619 c = lambda x: np.array(x, dtype=dtype) 620 621 value_0 = constant_op.constant(c([[4.0, 5.0]])) 622 value_1 = constant_op.constant(c(3.0)) 623 624 w0 = ta.write(0, value_0) 625 w1 = w0.write(1, value_1) 626 r0 = w1.read(0) 627 r1 = w1.read(1) 628 r0_2 = w1.read(0) 629 630 # Test individual components' gradients 631 grad_just_r0 = gradients_impl.gradients( 632 ys=[r0], xs=[value_0], grad_ys=[c([[2.0, 3.0]])]) 633 grad_just_r0_vals = session.run(grad_just_r0) 634 self.assertAllEqual(c([[2.0, 3.0]]), grad_just_r0_vals[0]) 635 636 grad_r0_r0_2 = gradients_impl.gradients( 637 ys=[r0, r0_2], 638 xs=[value_0], 639 grad_ys=[c([[2.0, 3.0]]), c([[1.0, -1.0]])]) 640 grad_r0_r0_2_vals = session.run(grad_r0_r0_2) 641 self.assertAllEqual(c([[3.0, 2.0]]), grad_r0_r0_2_vals[0]) 642 643 grad_just_r1 = gradients_impl.gradients( 644 ys=[r1], xs=[value_1], grad_ys=[c(-2.0)]) 645 grad_just_r1_vals = session.run(grad_just_r1) 646 self.assertAllEqual(c(-2.0), grad_just_r1_vals[0]) 647 648 # Test combined gradients 649 grad = gradients_impl.gradients( 650 ys=[r0, r0_2, r1], 651 xs=[value_0, value_1], 652 grad_ys=[c([[2.0, 3.0]]), c([[1.0, -1.0]]), c(-2.0)]) 653 grad_vals = session.run(grad) 654 self.assertEqual(len(grad_vals), 2) 655 self.assertAllEqual(c([[3.0, 2.0]]), grad_vals[0]) 656 self.assertAllEqual(c(-2.0), grad_vals[1]) 657 658 def testTensorArrayGradientWriteRead(self): 659 for dtype in (np.float32, np.float64, np.int32, np.int64, np.complex64, 660 np.complex128): 661 self._testTensorArrayGradientWriteReadType(dtype) 662 663 def _testTensorArrayGradientWritePackConcatAndRead(self): 664 with self.test_session(use_gpu=True) as sess: 665 ta = tensor_array_ops.TensorArray( 666 dtype=dtypes.float32, 667 tensor_array_name="foo", 668 size=2, 669 clear_after_read=False) 670 671 value_0 = constant_op.constant([-1.0, 1.0]) 672 value_1 = constant_op.constant([-10.0, 10.0]) 673 674 w0 = ta.write(0, value_0) 675 w1 = w0.write(1, value_1) 676 p0 = w1.stack() 677 r0 = w1.read(0) 678 s0 = w1.concat() 679 680 # Test gradient accumulation between read(0), pack(), and concat() 681 with ops.control_dependencies([p0, r0, s0]): 682 grad_r = gradients_impl.gradients( 683 ys=[p0, r0, s0], 684 xs=[value_0, value_1], 685 grad_ys=[ 686 [[2.0, 3.0], [4.0, 5.0]], # pack gradient 687 [-0.5, 1.5], # read(0) gradient 688 [20.0, 30.0, 40.0, 50.0] 689 ]) # concat gradient 690 grad_vals = sess.run(grad_r) # 2 + 2 entries 691 692 self.assertAllClose([2.0 - 0.5 + 20.0, 3.0 + 1.5 + 30.0], grad_vals[0]) 693 self.assertAllEqual([4.0 + 40.0, 5.0 + 50.0], grad_vals[1]) 694 695 def testTensorArrayGradientWritePackConcatAndRead(self): 696 self._testTensorArrayGradientWritePackConcatAndRead() 697 698 @test_util.run_in_graph_and_eager_modes() 699 def testTensorArrayReadTwice(self): 700 with self.test_session(use_gpu=True): 701 value = constant_op.constant([[1.0, -1.0], [10.0, -10.0]]) 702 703 ta_readonce = tensor_array_ops.TensorArray( 704 dtype=dtypes.float32, tensor_array_name="foo", size=2) 705 706 w_readonce = ta_readonce.unstack(value) 707 r0_readonce = w_readonce.read(0) 708 709 with self.assertRaisesOpError( 710 r"Could not read index 0 twice because it was cleared after a " 711 r"previous read \(perhaps try setting clear_after_read = false\?\)"): 712 with ops.control_dependencies([r0_readonce]): 713 self.evaluate(w_readonce.read(0)) 714 715 ta_readtwice = tensor_array_ops.TensorArray( 716 dtype=dtypes.float32, 717 tensor_array_name="foo", 718 size=2, 719 clear_after_read=False) 720 w_readtwice = ta_readtwice.unstack(value) 721 r0_readtwice = w_readtwice.read(0) 722 with ops.control_dependencies([r0_readtwice]): 723 r1_readtwice = w_readtwice.read(0) 724 725 self.assertAllEqual([1.0, -1.0], self.evaluate(r1_readtwice)) 726 727 def _testTensorArrayGradientUnpackRead(self): 728 with self.test_session(use_gpu=True) as session: 729 ta = tensor_array_ops.TensorArray( 730 dtype=dtypes.float32, 731 tensor_array_name="foo", 732 size=2, 733 clear_after_read=False) 734 735 value = constant_op.constant([[1.0, -1.0], [10.0, -10.0]]) 736 737 w = ta.unstack(value) 738 r0 = w.read(0) 739 r0_1 = w.read(0) 740 r1 = w.read(1) 741 742 # Test combined gradients + aggregation of read(0) 743 grad = gradients_impl.gradients( 744 ys=[r0, r0_1, r1], 745 xs=[value], 746 grad_ys=[[2.0, 3.0], [-1.5, 1.5], [4.0, 5.0]]) 747 grad_vals = session.run(grad) 748 749 self.assertEqual(len(grad_vals), 1) 750 self.assertAllEqual([[2.0 - 1.5, 3.0 + 1.5], [4.0, 5.0]], grad_vals[0]) 751 752 def testTensorArrayGradientUnpackRead(self): 753 self._testTensorArrayGradientUnpackRead() 754 755 def testTensorArrayGradientSplitConcat(self): 756 with self.test_session(use_gpu=True) as session: 757 ta = tensor_array_ops.TensorArray( 758 dtype=dtypes.float32, tensor_array_name="foo", size=2, 759 infer_shape=False) 760 761 value = constant_op.constant( 762 [[1.0, -1.0], [10.0, -10.0], [100.0, -100.0]]) 763 764 w = ta.split(value, [2, 1]) 765 r = w.concat() 766 767 # Test combined gradients 768 grad = gradients_impl.gradients( 769 ys=[r], 770 xs=[value], 771 grad_ys=[[[2.0, -2.0], [20.0, -20.0], [200.0, -200.0]]]) 772 grad_vals = session.run(grad) 773 774 self.assertEqual(len(grad_vals), 1) 775 self.assertAllEqual([[2.0, -2.0], [20.0, -20.0], [200.0, -200.0]], 776 grad_vals[0]) 777 778 def _testTensorArrayGradientDynamicUnpackRead(self): 779 with self.test_session(use_gpu=True) as session: 780 ta = tensor_array_ops.TensorArray( 781 dtype=dtypes.float32, 782 tensor_array_name="foo", 783 size=0, 784 dynamic_size=True) 785 786 value = constant_op.constant([[1.0, -1.0], [10.0, -10.0]]) 787 788 w = ta.unstack(value) 789 r0 = w.read(0) 790 r1 = w.read(1) 791 792 # Test combined gradients + aggregation of read(0) 793 grad = gradients_impl.gradients( 794 ys=[r0, r1], xs=[value], grad_ys=[[2.0, 3.0], [4.0, 5.0]]) 795 grad_vals = session.run(grad) 796 797 self.assertEqual(len(grad_vals), 1) 798 self.assertAllEqual([[2.0, 3.0], [4.0, 5.0]], grad_vals[0]) 799 800 def testTensorArrayGradientDynamicUnpackRead(self): 801 self._testTensorArrayGradientDynamicUnpackRead() 802 803 @test_util.run_in_graph_and_eager_modes() 804 def testCloseTensorArray(self): 805 with self.test_session(use_gpu=True): 806 ta = tensor_array_ops.TensorArray( 807 dtype=dtypes.float32, tensor_array_name="foo", size=3) 808 self.evaluate(ta.close()) 809 810 @test_util.run_in_graph_and_eager_modes() 811 def testSizeTensorArray(self): 812 with self.test_session(use_gpu=True): 813 ta = tensor_array_ops.TensorArray( 814 dtype=dtypes.float32, tensor_array_name="foo", size=3) 815 s = ta.size() 816 self.assertAllEqual(3, self.evaluate(s)) 817 818 @test_util.run_in_graph_and_eager_modes() 819 def testWriteCloseTensorArray(self): 820 with self.test_session(use_gpu=True): 821 ta = tensor_array_ops.TensorArray( 822 dtype=dtypes.float32, 823 tensor_array_name="foo", 824 size=3, 825 infer_shape=False) 826 w0 = ta.write(0, [[4.0, 5.0]]) 827 w1 = w0.write(1, [3.0]) 828 self.evaluate(w1.close()) # Expected to run without problems 829 830 def _testWhileLoopWritePackGradients(self, dynamic_size, dtype): 831 np_dtype = dtype.as_numpy_dtype 832 with self.test_session(use_gpu=True): 833 def func(v0, state0, var): 834 ta = tensor_array_ops.TensorArray( 835 dtype=dtype, 836 tensor_array_name="foo", 837 size=0 if dynamic_size else 3, 838 dynamic_size=dynamic_size) 839 time_0 = array_ops.identity(0) 840 841 def body(time, ta_t, state): 842 sliced = array_ops.slice( 843 v0, begin=array_ops.stack([time, 0]), size=[1, -1]) 844 sliced = array_ops.squeeze(sliced) 845 out = sliced + var + state 846 state += sliced 847 ta_t = ta_t.write(time, out) 848 return (time + 1, ta_t, state) 849 850 (unused_0, h_final, unused_2) = control_flow_ops.while_loop( 851 cond=lambda time, unused_1, unused_2: time < 3, 852 body=body, 853 loop_vars=(time_0, ta, state0), 854 shape_invariants=(time_0.get_shape(), tensor_shape.unknown_shape(), 855 tensor_shape.unknown_shape()), 856 parallel_iterations=3) 857 vout = h_final.stack() 858 return vout 859 860 v0 = array_ops.identity(np.arange(3 * 5, dtype=np_dtype).reshape(3, 5)) 861 state0 = array_ops.identity(np.array([1] * 5, dtype=np_dtype)) 862 init_val = np.arange(100, 105, dtype=np_dtype) 863 var = variable_scope.get_variable( 864 "var", 865 shape=init_val.shape, 866 dtype=np_dtype, 867 initializer=init_ops.constant_initializer(init_val)) 868 869 vout = func(v0, state0, var) 870 grad_val = -np.arange(3 * 5, dtype=np_dtype).reshape(3, 5) 871 if context.in_graph_mode(): 872 v0_grad = gradients_impl.gradients([vout], [v0], [grad_val])[0] 873 state0_grad = gradients_impl.gradients([vout], [state0], [grad_val])[0] 874 var_grad = gradients_impl.gradients([vout], [var], [grad_val])[0] 875 variables.global_variables_initializer().run() 876 else: 877 grad_fn = backprop.gradients_function(func) 878 v0_grad, state0_grad, var_grad = grad_fn(v0, state0, var, dy=grad_val) 879 880 state0_t, var_t, v0_t, vout_t, v0_grad_t, var_grad_t, state0_grad_t = ( 881 self.evaluate( 882 ([state0, var, v0, vout, v0_grad, var_grad, state0_grad]))) 883 just_v0_grad_t = self.evaluate(v0_grad) 884 885 # state = [ state0 | state0 + v0[0] | state0 + v0[0] + v0[1] ] 886 # vout = [ v0[0] + var + state[0] | 887 # v0[1] + var + state[1] | 888 # v0[2] + var + state[2] ] 889 # = [ v0[0] + var + state0 | 890 # v0[1] + var + state0 + v0[0] | 891 # v0[2] + var + state0 + v0[0] + v0[1] ] 892 # 893 # d(vout[0])/d(v0) = [1 | 0 | 0 ] 894 # d(vout[1])/d(v0) = [1 | 1 | 0 ] 895 # d(vout[2])/d(v0) = [1 | 1 | 1 ] 896 # d(vout)/d(var) = [1 | 1 | 1] 897 # d(vout)/d(state0) = [ 1 | 1 | 1 ] 898 899 state_per_time = np.array( 900 [state0_t, state0_t + v0_t[0, :], state0_t + v0_t[0, :] + v0_t[1, :]]) 901 902 # Compare forward prop 903 self.assertAllClose(v0_t + var_t + state_per_time, vout_t) 904 905 # Compare backward prop 906 expected_v0_grad_t = np.array([ 907 grad_val[0, :] + grad_val[1, :] + grad_val[2, :], 908 grad_val[1, :] + grad_val[2, :], grad_val[2, :] 909 ]) 910 911 self.assertAllEqual(expected_v0_grad_t, v0_grad_t) 912 self.assertAllEqual(expected_v0_grad_t, just_v0_grad_t) 913 self.assertAllClose(grad_val.sum(axis=0), var_grad_t) 914 self.assertAllClose(grad_val.sum(axis=0), state0_grad_t) 915 916 @test_util.run_in_graph_and_eager_modes() 917 def testWhileLoopWritePackGradients(self): 918 self._testWhileLoopWritePackGradients( 919 dynamic_size=False, dtype=dtypes.float32) 920 # TODO(ebrevdo): re-enable when While supports non-float32 gradients. 921 # self._testWhileLoopWritePackGradients( 922 # dynamic_size=False, dtype=tf.int64) 923 924 def testWhileLoopDynamicWritePackGradients(self): 925 self._testWhileLoopWritePackGradients( 926 dynamic_size=True, dtype=dtypes.float32) 927 928 @test_util.run_in_graph_and_eager_modes() 929 def testGradSerialTwoLoops(self): 930 with self.test_session(use_gpu=True): 931 def loop(x): 932 num_steps = 100 933 acc = tensor_array_ops.TensorArray( 934 dtype=dtypes.float32, 935 size=num_steps, 936 clear_after_read=False, 937 element_shape=tensor_shape.scalar()) 938 i = constant_op.constant(0, name="i") 939 940 c = lambda i, acc: i < 5 941 942 def b(i, acc): 943 x1 = control_flow_ops.cond( 944 math_ops.equal(i, 0), lambda: x, 945 lambda: math_ops.multiply(acc.read(i - 1), 2.0)) 946 return i + 1, acc.write(i, x1) 947 948 i1, acc1 = control_flow_ops.while_loop(c, b, [i, acc]) 949 950 z = constant_op.constant(0.0) 951 952 def fn(i, acc): 953 return i + 1, acc.write(i, z) 954 955 _, acc2 = control_flow_ops.while_loop(lambda i, acc: i < num_steps, fn, 956 [i1, acc1]) 957 958 r = acc2.stack() 959 return r 960 961 x = constant_op.constant(2.0, name="x") 962 if context.in_graph_mode(): 963 grad = gradients_impl.gradients(loop(x), [x])[0] 964 else: 965 grad = backprop.gradients_function(loop)(x)[0] 966 self.assertAllClose(31.0, self.evaluate(grad)) 967 968 def testSumOfTwoReadVariablesWithoutRepeatGrad(self): 969 with self.test_session(use_gpu=True) as session: 970 a = array_ops.identity( 971 np.arange( 972 3 * 5, dtype=np.float32).reshape(3, 5) + 1) 973 b = array_ops.identity( 974 np.arange( 975 3 * 5, dtype=np.float32).reshape(3, 5) + 1 + 3 * 5) 976 ta = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=2) 977 ta = ta.write(0, a, name="write_a") 978 ta = ta.write(1, b, name="write_b") 979 c = ( 980 ta.read( 981 0, name="read_a_0") + # a + b 982 ta.read( 983 1, name="read_b_0")) 984 g0 = -(np.arange(3 * 5, dtype=np.float32).reshape(3, 5) + 1) 985 grad_a = gradients_impl.gradients([c], [a], [g0])[0] # d(a+b)/da = 1 986 grad_b = gradients_impl.gradients([c], [b], [g0])[0] # d(a+b)/db = 1 987 988 # Test gradients calculated individually 989 grad_a_t, = session.run([grad_a]) 990 self.assertAllEqual(grad_a_t, g0) 991 992 grad_b_t, = session.run([grad_b]) 993 self.assertAllEqual(grad_b_t, g0) 994 995 # Test gradients calculated jointly 996 joint_grad_a_t, joint_grad_b_t = session.run([grad_a, grad_b]) 997 self.assertAllEqual(joint_grad_a_t, g0) 998 self.assertAllEqual(joint_grad_b_t, g0) 999 1000 def _grad_source_for_name(self, name): 1001 return tensor_array_grad._GetGradSource(constant_op.constant(0, name=name)) 1002 1003 def testGetGradSource_Invalid(self): 1004 with self.assertRaises(ValueError): 1005 self._grad_source_for_name("") 1006 with self.assertRaises(ValueError): 1007 self._grad_source_for_name("foo") 1008 with self.assertRaises(ValueError): 1009 self._grad_source_for_name("foo/bar") 1010 1011 def testGetGradSource_NoEnclosingScope(self): 1012 self.assertEqual("gradients:0", self._grad_source_for_name("gradients")) 1013 self.assertEqual("gradients_0:0", self._grad_source_for_name("gradients_0")) 1014 self.assertEqual("gradients", self._grad_source_for_name("gradients/foo")) 1015 self.assertEqual("gradients_0", 1016 self._grad_source_for_name("gradients_0/foo")) 1017 self.assertEqual("gradients", 1018 self._grad_source_for_name("gradients/foo/bar")) 1019 self.assertEqual("gradients_0", 1020 self._grad_source_for_name("gradients_0/foo/bar")) 1021 1022 def testGetGradSource_EnclosingScope(self): 1023 self.assertEqual("foo/gradients:0", 1024 self._grad_source_for_name("foo/gradients")) 1025 self.assertEqual("foo/gradients_0:0", 1026 self._grad_source_for_name("foo/gradients_0")) 1027 self.assertEqual("foo/gradients", 1028 self._grad_source_for_name("foo/gradients/bar")) 1029 self.assertEqual("foo/gradients_0", 1030 self._grad_source_for_name("foo/gradients_0/bar")) 1031 self.assertEqual("foo/bar/gradients", 1032 self._grad_source_for_name("foo/bar/gradients/baz")) 1033 self.assertEqual("foo/bar/gradients_0", 1034 self._grad_source_for_name("foo/bar/gradients_0/baz")) 1035 1036 def testGetGradSource_NestedUsesInnermost(self): 1037 self.assertEqual( 1038 "foo/gradients/bar/gradients_0", 1039 self._grad_source_for_name("foo/gradients/bar/gradients_0/baz")) 1040 1041 def testWriteShape(self): 1042 with self.test_session(use_gpu=True): 1043 ta = tensor_array_ops.TensorArray( 1044 dtype=dtypes.float32, tensor_array_name="foo", size=3) 1045 c0 = constant_op.constant([4.0, 5.0]) 1046 w0 = ta.write(0, c0) 1047 r0 = w0.read(0) 1048 self.assertAllEqual(c0.get_shape(), r0.get_shape()) 1049 1050 ta = tensor_array_ops.TensorArray( 1051 dtype=dtypes.float32, tensor_array_name="foo", size=3) 1052 c1 = constant_op.constant([6.0, 7.0]) 1053 w1 = w0.write(1, c1) 1054 r0 = w1.read(0) 1055 r1 = w1.read(1) 1056 self.assertAllEqual(c0.get_shape(), r0.get_shape()) 1057 self.assertAllEqual(c1.get_shape(), r1.get_shape()) 1058 1059 ta = tensor_array_ops.TensorArray( 1060 dtype=dtypes.float32, tensor_array_name="foo", size=3) 1061 c2 = constant_op.constant([4.0, 5.0, 6.0]) 1062 with self.assertRaises(ValueError): 1063 w0.write(0, c2) 1064 1065 def testPartlyUnknownShape(self): 1066 with self.test_session(use_gpu=True): 1067 ta = tensor_array_ops.TensorArray( 1068 dtype=dtypes.float32, tensor_array_name="foo", size=6) 1069 1070 c0 = array_ops.placeholder(dtypes.float32, [None, None, None, 3]) 1071 w0 = ta.write(0, c0) 1072 r0 = w0.read(0) 1073 self.assertAllEqual([None, None, None, 3], r0.get_shape().as_list()) 1074 1075 c1 = array_ops.placeholder(dtypes.float32, [None, None, None, 3]) 1076 w1 = w0.write(1, c1) 1077 r1 = w1.read(0) 1078 self.assertAllEqual([None, None, None, 3], r1.get_shape().as_list()) 1079 1080 # Writing less specific shape (doesn't change type.) 1081 c2 = array_ops.placeholder(dtypes.float32, [None, None, None, None]) 1082 w2 = w1.write(2, c2) 1083 r2 = w2.read(0) 1084 self.assertAllEqual([None, None, None, 3], r2.get_shape().as_list()) 1085 1086 # Writing more specific shape in one dimension and less specific in 1087 # another. 1088 c3 = array_ops.placeholder(dtypes.float32, [None, None, 2, None]) 1089 w3 = w2.write(3, c3) 1090 r3 = w3.read(0) 1091 self.assertAllEqual([None, None, 2, 3], r3.get_shape().as_list()) 1092 1093 # Writing partly defined shape using TensorArray.scatter. 1094 c4 = array_ops.placeholder(dtypes.float32, [2, None, 4, 2, 3]) 1095 w4 = w3.scatter([4, 5], c4) 1096 r4 = w4.read(0) 1097 self.assertAllEqual([None, 4, 2, 3], r4.get_shape().as_list()) 1098 1099 # Writing fully defined shape using TensorArray.split. 1100 c5 = array_ops.placeholder(dtypes.float32, [10, 4, 2, 3]) 1101 w5 = w4.split(c5, constant_op.constant([5, 5])) 1102 r5 = w5.read(0) 1103 self.assertAllEqual([5, 4, 2, 3], r5.get_shape().as_list()) 1104 1105 @test_util.run_in_graph_and_eager_modes() 1106 def _testUnpackShape(self): 1107 with self.test_session(use_gpu=True): 1108 ta = tensor_array_ops.TensorArray( 1109 dtype=dtypes.float32, 1110 tensor_array_name="foo", 1111 size=0, 1112 dynamic_size=True, 1113 infer_shape=True) 1114 value = constant_op.constant( 1115 [[1.0, -1.0], [10.0, -10.0], [100.0, -100.0]]) 1116 w0 = ta.unstack(value) 1117 r0 = w0.read(0) 1118 self.assertAllEqual((2,), r0.get_shape()) 1119 1120 c1 = constant_op.constant([4.0, 5.0]) 1121 w1 = w0.write(3, c1) 1122 1123 with self.assertRaisesOpError( 1124 r"Could not read index 0 twice because it was cleared after a " 1125 r"previous read \(perhaps try setting clear_after_read = false\?\)"): 1126 with ops.control_dependencies([r0]): 1127 self.evaluate(w1.read(0)) 1128 1129 r1 = w1.read(1) 1130 self.assertAllEqual(c1.get_shape(), r1.shape) 1131 1132 c2 = constant_op.constant([4.0, 5.0, 6.0]) 1133 with self.assertRaises(ValueError): 1134 w1.write(4, c2) 1135 1136 def testUnpackShape(self): 1137 self._testUnpackShape() 1138 1139 @test_util.run_in_graph_and_eager_modes() 1140 def testSplitShape(self): 1141 with self.test_session(use_gpu=True): 1142 ta = tensor_array_ops.TensorArray( 1143 dtype=dtypes.float32, 1144 tensor_array_name="foo", 1145 size=0, 1146 dynamic_size=True, 1147 infer_shape=True) 1148 value = constant_op.constant([[1.0, -1.0], [2.0, -2.0], [3.0, -3.0]]) 1149 w0 = ta.split(value, [1, 1, 1]) 1150 r0 = w0.read(0) 1151 self.assertAllEqual((1, 2), r0.get_shape()) 1152 1153 ta1 = tensor_array_ops.TensorArray( 1154 dtype=dtypes.float32, 1155 tensor_array_name="foo1", 1156 size=0, 1157 dynamic_size=True, 1158 infer_shape=True) 1159 w0 = ta1.split(value, [1, 2]) 1160 r0 = w0.read(0) 1161 if context.in_graph_mode(): 1162 self.assertEqual(r0.get_shape().ndims, None) 1163 self.assertEqual( 1164 tensor_shape.TensorShape( 1165 ta1.handle.op.get_attr("element_shape")).ndims, None) 1166 else: 1167 self.assertEqual((1, 2), r0.get_shape()) 1168 self.assertEqual((2, 2), w0.read(1).get_shape()) 1169 1170 def testWriteUnknownShape(self): 1171 with self.test_session(use_gpu=True): 1172 ta = tensor_array_ops.TensorArray( 1173 dtype=dtypes.float32, 1174 tensor_array_name="foo", 1175 size=3, 1176 infer_shape=True) 1177 c0 = array_ops.placeholder(dtypes.float32) 1178 w0 = ta.write(0, c0) 1179 r0 = w0.read(0) 1180 self.assertAllEqual(r0.get_shape(), tensor_shape.unknown_shape()) 1181 1182 def _testGradientWhenNotAllComponentsRead(self): 1183 with self.test_session(use_gpu=True) as session: 1184 ta = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=2) 1185 x = constant_op.constant([2.0, 3.0]) 1186 w = ta.unstack(x) 1187 r0 = w.read(0) 1188 # calculate (dr0/dx0, dr0/dx1). since r0 = x0, gradients are (1, 0). 1189 grad_r0 = gradients_impl.gradients(ys=[r0], xs=[x], grad_ys=[1.0]) 1190 grad_r0_vals = session.run(grad_r0)[0] 1191 self.assertAllEqual(grad_r0_vals, [1.0, 0.0]) 1192 1193 def testGradientWhenNotAllComponentsRead(self): 1194 self._testGradientWhenNotAllComponentsRead() 1195 1196 def _testTensorArrayUnpackDynamic(self): 1197 with self.test_session(use_gpu=True) as sess: 1198 ta = tensor_array_ops.TensorArray( 1199 dtype=dtypes.float32, size=3, dynamic_size=True) 1200 x = constant_op.constant([1.0, 2.0, 3.0]) 1201 w0 = ta.unstack(x) 1202 w1 = w0.write(3, 4.0) 1203 r = w1.stack() 1204 self.assertAllEqual(np.array([1.0, 2.0, 3.0, 4.0]), r.eval()) 1205 grad = gradients_impl.gradients(ys=[r], xs=[x]) 1206 self.assertAllEqual(np.array([1.0, 1.0, 1.0]), sess.run(grad)[0]) 1207 1208 def testTensorArrayUnpackDynamic(self): 1209 self._testTensorArrayUnpackDynamic() 1210 1211 def testTensorArraySplitDynamic(self): 1212 with self.test_session(use_gpu=True) as sess: 1213 ta = tensor_array_ops.TensorArray( 1214 dtype=dtypes.float32, size=3, dynamic_size=True) 1215 x = constant_op.constant([1.0, 2.0, 3.0]) 1216 w0 = ta.split(x, [1, 1, 1]) 1217 w1 = w0.write(3, [4.0]) 1218 r = w1.concat() 1219 self.assertAllEqual(np.array([1.0, 2.0, 3.0, 4.0]), r.eval()) 1220 grad = gradients_impl.gradients(ys=[r], xs=[x]) 1221 self.assertAllEqual(np.array([1.0, 1.0, 1.0]), sess.run(grad)[0]) 1222 1223 def _testTensorArrayEvalEmpty(self): 1224 with self.test_session(use_gpu=True): 1225 ta = tensor_array_ops.TensorArray( 1226 dtype=dtypes.float32, size=0, dynamic_size=False, infer_shape=False) 1227 with self.assertRaisesOpError( 1228 "TensorArray has size zero, but element shape <unknown> is not fully " 1229 "defined. Currently only static shapes are supported when packing " 1230 "zero-size TensorArrays."): 1231 ta.stack().eval() 1232 1233 def testTensorArrayEvalEmpty(self): 1234 self._testTensorArrayEvalEmpty() 1235 1236 # this test is ill-defined for Eager mode --- unpacking an empty tensor 1237 # gives an empty list / there is not equivalent of "mark_used" in Eager 1238 def _testTensorArrayEvalEmptyWithDefault(self): 1239 with self.test_session(use_gpu=True): 1240 ta = tensor_array_ops.TensorArray( 1241 dtype=dtypes.float32, size=0, dynamic_size=False, infer_shape=True) 1242 self.assertEqual(0, ta.size().eval()) 1243 # Don't actually perform the pack. This stores the static shape. 1244 ta.unstack(array_ops.zeros([0, 3, 5])).mark_used() 1245 packed = ta.stack() 1246 concatenated = ta.concat() 1247 self.assertAllEqual([0, 3, 5], packed.eval().shape) 1248 # Concatenating zero tensors along their first dimension gives a 1249 # first dimension of zero 1250 self.assertAllEqual([0, 5], concatenated.eval().shape) 1251 1252 def testTensorArrayEvalEmptyWithDefault(self): 1253 self._testTensorArrayEvalEmptyWithDefault() 1254 1255 def testTensorArrayScatterReadAndGradients(self): 1256 with self.test_session(use_gpu=True) as session: 1257 ta = tensor_array_ops.TensorArray( 1258 dtype=dtypes.float32, 1259 tensor_array_name="foo", 1260 size=0, 1261 dynamic_size=True) 1262 1263 indices = constant_op.constant([1, 8]) 1264 value = constant_op.constant([[1.0, -1.0], [10.0, -10.0]]) 1265 1266 w = ta.scatter(indices, value) 1267 r0 = w.read(1) 1268 r1 = w.read(8) 1269 1270 # Test combined gradients + aggregation of read(0) 1271 grad = gradients_impl.gradients( 1272 ys=[r0, r1], xs=[value], grad_ys=[[2.0, 3.0], [4.0, 5.0]]) 1273 read_vals, grad_vals = session.run([[r0, r1], grad]) 1274 1275 self.assertEqual(len(read_vals), 2) 1276 self.assertEqual(len(grad_vals), 1) 1277 self.assertAllEqual([1.0, -1.0], read_vals[0]) 1278 self.assertAllEqual([10.0, -10.0], read_vals[1]) 1279 self.assertAllEqual([[2.0, 3.0], [4.0, 5.0]], grad_vals[0]) 1280 1281 @test_util.run_in_graph_and_eager_modes() 1282 def testTensorArrayWriteGatherAndGradients(self): 1283 with self.test_session(use_gpu=True) as session: 1284 ta = tensor_array_ops.TensorArray( 1285 dtype=dtypes.float32, 1286 tensor_array_name="foo", 1287 size=0, 1288 dynamic_size=True) 1289 1290 def func(values): 1291 indices = constant_op.constant([1, 8]) 1292 w = ta.unstack(values) 1293 g = w.gather(indices) 1294 return g 1295 1296 values = constant_op.constant([[1.0 * x, -1.0 * x] for x in range(10)]) 1297 g = func(values) 1298 grad_ys = [[[2.0, 3.0], [4.0, 5.0]]] 1299 # Test combined gradients + aggregation of read(0) 1300 if context.in_graph_mode(): 1301 grad = gradients_impl.gradients(ys=[g], xs=[values], grad_ys=grad_ys) 1302 g_vals, grad_vals = session.run([[g], grad]) 1303 else: 1304 g_vals = [g] 1305 grad_vals = backprop.gradients_function(func)( 1306 values, dy=constant_op.constant(grad_ys[0], dtype=dtypes.float32)) 1307 1308 # Gradients for 8 of the 10 unread components are zero. 1309 expected_grad = np.zeros((10, 2)) 1310 expected_grad[1] = [2.0, 3.0] 1311 expected_grad[8] = [4.0, 5.0] 1312 1313 self.assertEqual(len(g_vals), 1) 1314 self.assertEqual(len(grad_vals), 1) 1315 self.assertAllEqual([[1.0, -1.0], [8.0, -8.0]], g_vals[0]) 1316 self.assertAllEqual(expected_grad, grad_vals[0]) 1317 1318 def testTensorArrayGetsDeviceFromFirstWrite(self): 1319 with ops.device("/job:worker/task:0/cpu:0"): 1320 # this initial device will be ignored. 1321 ta = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=2) 1322 with ops.device("/job:worker/task:1/cpu:0"): 1323 # the first write sets the op's device. 1324 ta = ta.write(0, 1.0) 1325 with ops.device("/job:worker/task:2/cpu:0"): 1326 # subsequent writes do not modify the op's device. 1327 ta = ta.write(1, 1.0) 1328 1329 # The gradient TA will sit on the same device as the forward TA. 1330 ta_grad = ta.grad("grad") 1331 flows = [ta.flow, ta_grad.flow] 1332 1333 # Similar tests for unpack and split 1334 with ops.device("/job:worker/task:0/cpu:0"): 1335 ta = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=3) 1336 with ops.device("/job:worker/task:1/cpu:0"): 1337 ta = ta.unstack([1.0, 2.0]) 1338 with ops.device("/job:worker/task:2/cpu:0"): 1339 ta = ta.write(2, 3.0) 1340 flows.append(ta.flow) 1341 1342 with ops.device("/job:worker/task:0/cpu:0"): 1343 ta = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=2) 1344 with ops.device("/job:worker/task:1/cpu:0"): 1345 ta = ta.split([1.0, 2.0], [1, 1]) 1346 flows.append(ta.flow) 1347 1348 session = session_lib.Session(self._workers[0].target) 1349 1350 run_options = config_pb2.RunOptions( 1351 trace_level=config_pb2.RunOptions.FULL_TRACE) 1352 run_metadata = config_pb2.RunMetadata() 1353 1354 session.run(flows, options=run_options, run_metadata=run_metadata) 1355 self.assertTrue(run_metadata.HasField("step_stats")) 1356 dev_stats = {d.device: d.node_stats 1357 for d in run_metadata.step_stats.dev_stats} 1358 for d in dev_stats: 1359 if "/task:1/" in d: 1360 self.assertTrue( 1361 [s for s in dev_stats[d] if "/TensorArray" in s.node_name]) 1362 else: 1363 self.assertFalse( 1364 [s for s in dev_stats[d] if "/TensorArray" in s.node_name]) 1365 1366 def testTensorArrayGetsDeviceFromFirstWriteInWhileLoop(self): 1367 with ops.device("/job:worker/task:0/cpu:0"): 1368 ta = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=2) 1369 1370 def _body(i, ta_i): 1371 with ops.device("/job:worker/task:1/cpu:0"): 1372 return i + 1, ta_i.write(i, constant_op.constant(0.0)) 1373 1374 _, ta_out = control_flow_ops.while_loop( 1375 lambda i, ta: i < 2, _body, loop_vars=[0, ta]) 1376 1377 session = session_lib.Session(self._workers[0].target) 1378 1379 run_options = config_pb2.RunOptions( 1380 trace_level=config_pb2.RunOptions.FULL_TRACE) 1381 run_metadata = config_pb2.RunMetadata() 1382 1383 session.run(ta_out.flow, options=run_options, run_metadata=run_metadata) 1384 self.assertTrue(run_metadata.HasField("step_stats")) 1385 dev_stats = {d.device: d.node_stats 1386 for d in run_metadata.step_stats.dev_stats} 1387 for d in dev_stats: 1388 if "/task:1/" in d: 1389 self.assertTrue( 1390 [s for s in dev_stats[d] if "/TensorArray" in s.node_name]) 1391 else: 1392 self.assertFalse( 1393 [s for s in dev_stats[d] if "/TensorArray" in s.node_name]) 1394 1395 def testTensorArrayDisabledColocateWithFirstWriteCall(self): 1396 with ops.device("/job:worker/task:0/cpu:0"): 1397 ta = tensor_array_ops.TensorArray( 1398 dtype=dtypes.float32, size=2, colocate_with_first_write_call=False) 1399 1400 def _body(i, ta_i): 1401 with ops.device("/job:worker/task:1/cpu:0"): 1402 return i + 1, ta_i.write(i, constant_op.constant(0.0)) 1403 1404 _, ta_out = control_flow_ops.while_loop( 1405 lambda i, ta: i < 2, _body, loop_vars=[0, ta]) 1406 1407 session = session_lib.Session(self._workers[0].target) 1408 1409 run_options = config_pb2.RunOptions( 1410 trace_level=config_pb2.RunOptions.FULL_TRACE) 1411 run_metadata = config_pb2.RunMetadata() 1412 1413 session.run(ta_out.flow, options=run_options, run_metadata=run_metadata) 1414 self.assertTrue(run_metadata.HasField("step_stats")) 1415 dev_stats = {d.device: list(d.node_stats) 1416 for d in run_metadata.step_stats.dev_stats} 1417 for d in dev_stats: 1418 if "/task:0/" in d and "CPU" in d: # Skip any GPU node stats 1419 self.assertTrue( 1420 [s for s in dev_stats[d] if "/TensorArray" in s.node_name]) 1421 else: 1422 self.assertFalse( 1423 [s for s in dev_stats[d] if "/TensorArray" in s.node_name]) 1424 1425 @test_util.run_in_graph_and_eager_modes() 1426 def testTensorArrayIdentity(self): 1427 with self.test_session(use_gpu=True): 1428 ta0 = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=2, 1429 infer_shape=False) 1430 ta1 = tensor_array_ops.TensorArray(dtype=dtypes.int32, size=4, 1431 infer_shape=True) 1432 1433 ta0 = ta0.write(0, 0.) 1434 ta1 = ta1.write(0, 1) 1435 1436 v0 = variable_scope.get_variable( 1437 "v0", shape=(), initializer=init_ops.zeros_initializer()) 1438 v1 = variable_scope.get_variable( 1439 "v1", shape=(), initializer=init_ops.zeros_initializer()) 1440 1441 with ops.control_dependencies([v0.assign_add(1)]): 1442 ta0 = ta0.identity() 1443 1444 with ops.control_dependencies([v1.assign_add(1)]): 1445 ta1 = ta1.identity() 1446 1447 read0 = ta0.read(0) 1448 read1 = ta1.read(0) 1449 1450 size0 = ta0.size() 1451 size1 = ta1.size() 1452 1453 # Tests correct properties on new TensorArrays. 1454 self.assertEqual(dtypes.float32, ta0.dtype) 1455 self.assertEqual(dtypes.int32, ta1.dtype) 1456 if context.in_graph_mode(): 1457 self.assertEqual(tensor_shape.unknown_shape(), read0.get_shape()) 1458 else: 1459 self.assertEqual(tensor_shape.scalar(), read1.get_shape()) 1460 self.assertEqual(tensor_shape.scalar(), read1.get_shape()) 1461 1462 if context.in_graph_mode(): 1463 variables.global_variables_initializer().run() 1464 1465 read0_v, read1_v, size0_v, size1_v = self.evaluate((read0, read1, size0, 1466 size1)) 1467 1468 # Tests that the control dependencies was added and executed. 1469 self.assertEqual(1, self.evaluate(v0)) 1470 self.assertEqual(1, self.evaluate(v1)) 1471 1472 # Tests correct TensorArray. 1473 self.assertEqual(read0_v, 0) 1474 self.assertEqual(read1_v, 1) 1475 self.assertEqual(size0_v, 2) 1476 self.assertEqual(size1_v, 4) 1477 1478 def testTensorArrayGradYsInCorrectScope(self): 1479 n_time = 1 1480 n_dim = 1 1481 x = constant_op.constant([[1.42]]) 1482 dy = constant_op.constant([[2.42]]) 1483 1484 ta = tensor_array_ops.TensorArray( 1485 dtypes.float32, size=n_time, element_shape=[n_dim]) 1486 for t in range(n_time): 1487 ta = ta.write(index=t, value=x[t]) 1488 y = ta.stack() 1489 # dy is outside of the gradients name scope; tf.gradients must 1490 # wrap it in the correct name scope. 1491 dx, = gradients_impl.gradients(ys=[y], xs=[x], grad_ys=[dy]) 1492 with self.test_session(use_gpu=True) as sess: 1493 vdx, vdy = sess.run([dx, dy]) 1494 self.assertAllClose(vdx, vdy) 1495 1496 1497if __name__ == "__main__": 1498 test.main() 1499