inputbuffer_test.cc revision 2f3328b03f2ef261781b02ca3571688356a7915c
1c8b59c046895fa5b6d79f73e0b5817330fcfbfc1A. Unique TensorFlower/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 29c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath Kudlur 39c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath KudlurLicensed under the Apache License, Version 2.0 (the "License"); 49c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath Kudluryou may not use this file except in compliance with the License. 59c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath KudlurYou may obtain a copy of the License at 69c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath Kudlur 79c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath Kudlur http://www.apache.org/licenses/LICENSE-2.0 89c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath Kudlur 99c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath KudlurUnless required by applicable law or agreed to in writing, software 109c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath Kudlurdistributed under the License is distributed on an "AS IS" BASIS, 119c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath KudlurWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 129c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath KudlurSee the License for the specific language governing permissions and 139c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath Kudlurlimitations under the License. 149c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath Kudlur==============================================================================*/ 159c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath Kudlur 16f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur#include "tensorflow/core/lib/io/inputbuffer.h" 17f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 18b481783fe0e00a86f6feb20a8dcad5fc4fc936a4Josh Levenberg#include <vector> 193a7d603feaf6434ffd6d9fd27cac65c1ef565e07Josh Levenberg#include "tensorflow/core/platform/env.h" 20f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 212f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang#include "tensorflow/core/lib/core/coding.h" 2256313def004795f75ef8281a0294c958d28f1e06Vijay Vasudevan#include "tensorflow/core/lib/core/errors.h" 233a7d603feaf6434ffd6d9fd27cac65c1ef565e07Josh Levenberg#include "tensorflow/core/lib/core/status.h" 24f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur#include "tensorflow/core/lib/core/status_test_util.h" 2556313def004795f75ef8281a0294c958d28f1e06Vijay Vasudevan#include "tensorflow/core/lib/strings/strcat.h" 2656313def004795f75ef8281a0294c958d28f1e06Vijay Vasudevan#include "tensorflow/core/platform/logging.h" 27f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur#include "tensorflow/core/platform/test.h" 28f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 29f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlurnamespace tensorflow { 3094054e083e03b2e7a298a0efbee560746f04d470Zongheng Yangnamespace { 31f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 32f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlurstatic std::vector<int> BufferSizes() { 33f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur return {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 34f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 12, 13, 14, 15, 16, 17, 18, 19, 20, 65536}; 35f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur} 36f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 37f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath KudlurTEST(InputBuffer, ReadLine_Empty) { 38f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur Env* env = Env::Default(); 39f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string fname = testing::TmpDir() + "/inputbuffer_test"; 40f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur WriteStringToFile(env, fname, ""); 41f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 42f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur for (auto buf_size : BufferSizes()) { 436fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan std::unique_ptr<RandomAccessFile> file; 44f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 45f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string line; 466fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan io::InputBuffer in(file.get(), buf_size); 47f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.ReadLine(&line))); 48f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur } 49f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur} 50f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 51f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath KudlurTEST(InputBuffer, ReadLine1) { 52f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur Env* env = Env::Default(); 53f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string fname = testing::TmpDir() + "/inputbuffer_test"; 54f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur WriteStringToFile(env, fname, "line one\nline two\nline three\n"); 55f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 56f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur for (auto buf_size : BufferSizes()) { 576fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan std::unique_ptr<RandomAccessFile> file; 58f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 59f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string line; 606fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan io::InputBuffer in(file.get(), buf_size); 61f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 62f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, "line one"); 63f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 64f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, "line two"); 65f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 66f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, "line three"); 67f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.ReadLine(&line))); 68f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur // A second call should also return end of file 69f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.ReadLine(&line))); 70f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur } 71f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur} 72f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 73f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath KudlurTEST(InputBuffer, ReadLine_NoTrailingNewLine) { 74f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur Env* env = Env::Default(); 75f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string fname = testing::TmpDir() + "/inputbuffer_test"; 76f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur WriteStringToFile(env, fname, "line one\nline two\nline three"); 77f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 78f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur for (auto buf_size : BufferSizes()) { 796fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan std::unique_ptr<RandomAccessFile> file; 80f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 81f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string line; 826fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan io::InputBuffer in(file.get(), buf_size); 83f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 84f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, "line one"); 85f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 86f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, "line two"); 87f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 88f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, "line three"); 89f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.ReadLine(&line))); 90f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur // A second call should also return end of file 91f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.ReadLine(&line))); 92f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur } 93f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur} 94f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 95f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath KudlurTEST(InputBuffer, ReadLine_EmptyLines) { 96f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur Env* env = Env::Default(); 97f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string fname = testing::TmpDir() + "/inputbuffer_test"; 98f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur WriteStringToFile(env, fname, "line one\n\n\nline two\nline three"); 99f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 100f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur for (auto buf_size : BufferSizes()) { 1016fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan std::unique_ptr<RandomAccessFile> file; 102f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 103f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string line; 1046fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan io::InputBuffer in(file.get(), buf_size); 105f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 106f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, "line one"); 107f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 108f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, ""); 109f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 110f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, ""); 111f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 112f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, "line two"); 113f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 114f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, "line three"); 115f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.ReadLine(&line))); 116f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur // A second call should also return end of file 117f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.ReadLine(&line))); 118f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur } 119f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur} 120f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 1211c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlowerTEST(InputBuffer, ReadLine_CRLF) { 1221c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower Env* env = Env::Default(); 1231c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower string fname = testing::TmpDir() + "/inputbuffer_test"; 1241c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower WriteStringToFile(env, fname, "line one\r\n\r\n\r\nline two\r\nline three"); 1251c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower 1261c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower for (auto buf_size : BufferSizes()) { 1276fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan std::unique_ptr<RandomAccessFile> file; 1281c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 1291c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower string line; 1306fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan io::InputBuffer in(file.get(), buf_size); 1311c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower TF_CHECK_OK(in.ReadLine(&line)); 1321c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower EXPECT_EQ(line, "line one"); 1331c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower TF_CHECK_OK(in.ReadLine(&line)); 1341c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower EXPECT_EQ(line, ""); 1351c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower TF_CHECK_OK(in.ReadLine(&line)); 1361c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower EXPECT_EQ(line, ""); 1371c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower TF_CHECK_OK(in.ReadLine(&line)); 1381c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower EXPECT_EQ(line, "line two"); 1391c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower TF_CHECK_OK(in.ReadLine(&line)); 1401c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower EXPECT_EQ(line, "line three"); 1411c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower EXPECT_TRUE(errors::IsOutOfRange(in.ReadLine(&line))); 1421c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower // A second call should also return end of file 1431c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower EXPECT_TRUE(errors::IsOutOfRange(in.ReadLine(&line))); 1441c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower } 1451c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower} 1461c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower 147f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath KudlurTEST(InputBuffer, ReadNBytes) { 148f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur Env* env = Env::Default(); 149f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string fname = testing::TmpDir() + "/inputbuffer_test"; 150f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur WriteStringToFile(env, fname, "0123456789"); 151f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 15294054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang // ReadNBytes(int64, string*). 153f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur for (auto buf_size : BufferSizes()) { 1546fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan std::unique_ptr<RandomAccessFile> file; 155f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 156f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string read; 1576fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan io::InputBuffer in(file.get(), buf_size); 158f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(0, in.Tell()); 159f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadNBytes(3, &read)); 160f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(read, "012"); 161f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(3, in.Tell()); 162f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadNBytes(0, &read)); 163f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(read, ""); 164f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(3, in.Tell()); 165f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadNBytes(4, &read)); 166f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(read, "3456"); 167f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(7, in.Tell()); 168f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadNBytes(0, &read)); 169f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(read, ""); 170f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(7, in.Tell()); 171f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.ReadNBytes(5, &read))); 172f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(read, "789"); 173f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(10, in.Tell()); 174f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.ReadNBytes(5, &read))); 175f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(read, ""); 176f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(10, in.Tell()); 177f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadNBytes(0, &read)); 178f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(read, ""); 179f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(10, in.Tell()); 180f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur } 18194054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang // ReadNBytes(int64, char*, size_t*). 18294054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang size_t bytes_read; 18394054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang for (auto buf_size : BufferSizes()) { 18494054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang std::unique_ptr<RandomAccessFile> file; 18594054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 18694054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang char read[5]; 18794054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang io::InputBuffer in(file.get(), buf_size); 18894054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 18994054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(0, in.Tell()); 19094054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_ASSERT_OK(in.ReadNBytes(3, read, &bytes_read)); 19194054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(StringPiece(read, 3), "012"); 19294054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 19394054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(3, in.Tell()); 19494054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_ASSERT_OK(in.ReadNBytes(0, read, &bytes_read)); 19594054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(StringPiece(read, 3), "012"); 19694054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 19794054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(3, in.Tell()); 19894054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_ASSERT_OK(in.ReadNBytes(4, read, &bytes_read)); 19994054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(StringPiece(read, 4), "3456"); 20094054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 20194054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(7, in.Tell()); 20294054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_ASSERT_OK(in.ReadNBytes(0, read, &bytes_read)); 20394054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(StringPiece(read, 4), "3456"); 20494054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 20594054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(7, in.Tell()); 20694054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_TRUE(errors::IsOutOfRange(in.ReadNBytes(5, read, &bytes_read))); 20794054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(StringPiece(read, 3), "789"); 20894054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 20994054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(10, in.Tell()); 21094054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_TRUE(errors::IsOutOfRange(in.ReadNBytes(5, read, &bytes_read))); 21194054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(StringPiece(read, 3), "789"); 21294054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 21394054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(10, in.Tell()); 21494054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_ASSERT_OK(in.ReadNBytes(0, read, &bytes_read)); 21594054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(StringPiece(read, 3), "789"); 21694054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(10, in.Tell()); 21794054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang } 218f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur} 219f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 220f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath KudlurTEST(InputBuffer, SkipNBytes) { 221f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur Env* env = Env::Default(); 222f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string fname = testing::TmpDir() + "/inputbuffer_test"; 223f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur WriteStringToFile(env, fname, "0123456789"); 224f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 225f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur for (auto buf_size : BufferSizes()) { 2266fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan std::unique_ptr<RandomAccessFile> file; 227f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 228f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string read; 2296fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan io::InputBuffer in(file.get(), buf_size); 230f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(0, in.Tell()); 231f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.SkipNBytes(3)); 232f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(3, in.Tell()); 233f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.SkipNBytes(0)); 234f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(3, in.Tell()); 235f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadNBytes(2, &read)); 236f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(read, "34"); 237f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(5, in.Tell()); 238f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.SkipNBytes(0)); 239f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(5, in.Tell()); 240f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.SkipNBytes(2)); 241f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(7, in.Tell()); 242f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadNBytes(1, &read)); 243f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(read, "7"); 244f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(8, in.Tell()); 245f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.SkipNBytes(5))); 246f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(10, in.Tell()); 247f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.SkipNBytes(5))); 248f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(10, in.Tell()); 249f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.ReadNBytes(5, &read))); 250f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(read, ""); 251f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(10, in.Tell()); 252f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur } 253f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur} 254f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 25594054e083e03b2e7a298a0efbee560746f04d470Zongheng YangTEST(InputBuffer, Seek) { 25694054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang Env* env = Env::Default(); 25794054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang string fname = testing::TmpDir() + "/inputbuffer_test"; 25894054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang WriteStringToFile(env, fname, "0123456789"); 25994054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 26094054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang for (auto buf_size : BufferSizes()) { 26194054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang std::unique_ptr<RandomAccessFile> file; 26294054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 26394054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang string read; 26494054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang io::InputBuffer in(file.get(), buf_size); 26594054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 26694054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(in.ReadNBytes(3, &read)); 26794054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(read, "012"); 26894054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(in.ReadNBytes(3, &read)); 26994054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(read, "345"); 27094054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 27194054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(in.Seek(0)); 27294054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(in.ReadNBytes(3, &read)); 27394054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(read, "012"); 27494054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 27594054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(in.Seek(3)); 27694054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(in.ReadNBytes(4, &read)); 27794054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(read, "3456"); 27894054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 27994054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(in.Seek(4)); 28094054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(in.ReadNBytes(4, &read)); 28194054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(read, "4567"); 28294054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 28394054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(in.Seek(1 << 25)); 28494054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_TRUE(errors::IsOutOfRange(in.ReadNBytes(1, &read))); 28594054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 28694054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_TRUE( 28794054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang StringPiece(in.Seek(-1).ToString()).contains("negative position")); 28894054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang } 28994054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang} 29094054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 2912f3328b03f2ef261781b02ca3571688356a7915cZongheng YangTEST(InputBuffer, ReadVarint32) { 2922f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang Env* env = Env::Default(); 2932f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang string fname = testing::TmpDir() + "/inputbuffer_test"; 2942f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang 2952f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang // Generates data. 2962f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang std::vector<uint32> data; 2972f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang uint32 i = 0; 2982f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang for (; i < (1 << 10); i += 1) data.push_back(i); 2992f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang for (; i < (1 << 15); i += 5) data.push_back(i); 3002f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang for (; i < (1 << 31); i += 132817) data.push_back(i); 3012f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang data.push_back(std::numeric_limits<uint32>::max()); 3022f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang 3032f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang // Writes the varints. 3042f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang { 3052f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang std::unique_ptr<WritableFile> file; 3062f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang TF_CHECK_OK(env->NewWritableFile(fname, &file)); 3072f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang string varint; 3082f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang for (uint32 number : data) { 3092f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang varint.clear(); 3102f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang core::PutVarint32(&varint, number); 3112f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang TF_CHECK_OK(file->Append(StringPiece(varint))); 3122f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang } 3132f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang } 3142f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang 3152f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang for (auto buf_size : BufferSizes()) { 3162f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang std::unique_ptr<RandomAccessFile> file; 3172f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 3182f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang io::InputBuffer in(file.get(), buf_size); 3192f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang uint32 result = 0; 3202f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang 3212f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang for (uint32 expected : data) { 3222f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang TF_ASSERT_OK(in.ReadVarint32(&result)); 3232f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang EXPECT_EQ(expected, result); 3242f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang } 3252f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang EXPECT_TRUE(errors::IsOutOfRange(in.ReadVarint32(&result))); 3262f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang } 3272f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang} 3282f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang 32994054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang} // namespace 330f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur} // namespace tensorflow 331