1c8b59c046895fa5b6d79f73e0b5817330fcfbfc1A. Unique TensorFlower/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. 29c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath Kudlur 39c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath KudlurLicensed under the Apache License, Version 2.0 (the "License"); 49c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath Kudluryou may not use this file except in compliance with the License. 59c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath KudlurYou may obtain a copy of the License at 69c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath Kudlur 79c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath Kudlur http://www.apache.org/licenses/LICENSE-2.0 89c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath Kudlur 99c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath KudlurUnless required by applicable law or agreed to in writing, software 109c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath Kudlurdistributed under the License is distributed on an "AS IS" BASIS, 119c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath KudlurWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 129c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath KudlurSee the License for the specific language governing permissions and 139c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath Kudlurlimitations under the License. 149c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath Kudlur==============================================================================*/ 159c3043ff3bf31a6a81810b4ce9e87ef936f1f529Manjunath Kudlur 16f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur#include "tensorflow/core/lib/io/inputbuffer.h" 17f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 18b481783fe0e00a86f6feb20a8dcad5fc4fc936a4Josh Levenberg#include <vector> 193a7d603feaf6434ffd6d9fd27cac65c1ef565e07Josh Levenberg#include "tensorflow/core/platform/env.h" 20f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 212f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang#include "tensorflow/core/lib/core/coding.h" 2256313def004795f75ef8281a0294c958d28f1e06Vijay Vasudevan#include "tensorflow/core/lib/core/errors.h" 233a7d603feaf6434ffd6d9fd27cac65c1ef565e07Josh Levenberg#include "tensorflow/core/lib/core/status.h" 24f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur#include "tensorflow/core/lib/core/status_test_util.h" 2556313def004795f75ef8281a0294c958d28f1e06Vijay Vasudevan#include "tensorflow/core/lib/strings/strcat.h" 2656313def004795f75ef8281a0294c958d28f1e06Vijay Vasudevan#include "tensorflow/core/platform/logging.h" 27f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur#include "tensorflow/core/platform/test.h" 28f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 29f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlurnamespace tensorflow { 3094054e083e03b2e7a298a0efbee560746f04d470Zongheng Yangnamespace { 31f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 32f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlurstatic std::vector<int> BufferSizes() { 33f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur return {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 34f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 12, 13, 14, 15, 16, 17, 18, 19, 20, 65536}; 35f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur} 36f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 37f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath KudlurTEST(InputBuffer, ReadLine_Empty) { 38f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur Env* env = Env::Default(); 39f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string fname = testing::TmpDir() + "/inputbuffer_test"; 40bc225bfaa534acc25047fe844f19edc333b7a76aPeter Hawkins TF_ASSERT_OK(WriteStringToFile(env, fname, "")); 41f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 42f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur for (auto buf_size : BufferSizes()) { 436fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan std::unique_ptr<RandomAccessFile> file; 44f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 45f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string line; 466fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan io::InputBuffer in(file.get(), buf_size); 47f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.ReadLine(&line))); 48f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur } 49f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur} 50f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 51f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath KudlurTEST(InputBuffer, ReadLine1) { 52f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur Env* env = Env::Default(); 53f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string fname = testing::TmpDir() + "/inputbuffer_test"; 54bc225bfaa534acc25047fe844f19edc333b7a76aPeter Hawkins TF_CHECK_OK( 55bc225bfaa534acc25047fe844f19edc333b7a76aPeter Hawkins WriteStringToFile(env, fname, "line one\nline two\nline three\n")); 56f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 57f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur for (auto buf_size : BufferSizes()) { 586fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan std::unique_ptr<RandomAccessFile> file; 59f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 60f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string line; 616fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan io::InputBuffer in(file.get(), buf_size); 62f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 63f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, "line one"); 64f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 65f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, "line two"); 66f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 67f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, "line three"); 68f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.ReadLine(&line))); 69f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur // A second call should also return end of file 70f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.ReadLine(&line))); 71f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur } 72f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur} 73f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 74f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath KudlurTEST(InputBuffer, ReadLine_NoTrailingNewLine) { 75f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur Env* env = Env::Default(); 76f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string fname = testing::TmpDir() + "/inputbuffer_test"; 77bc225bfaa534acc25047fe844f19edc333b7a76aPeter Hawkins TF_ASSERT_OK(WriteStringToFile(env, fname, "line one\nline two\nline three")); 78f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 79f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur for (auto buf_size : BufferSizes()) { 806fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan std::unique_ptr<RandomAccessFile> file; 81f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 82f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string line; 836fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan io::InputBuffer in(file.get(), buf_size); 84f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 85f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, "line one"); 86f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 87f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, "line two"); 88f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 89f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, "line three"); 90f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.ReadLine(&line))); 91f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur // A second call should also return end of file 92f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.ReadLine(&line))); 93f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur } 94f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur} 95f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 96f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath KudlurTEST(InputBuffer, ReadLine_EmptyLines) { 97f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur Env* env = Env::Default(); 98f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string fname = testing::TmpDir() + "/inputbuffer_test"; 99bc225bfaa534acc25047fe844f19edc333b7a76aPeter Hawkins TF_CHECK_OK( 100bc225bfaa534acc25047fe844f19edc333b7a76aPeter Hawkins WriteStringToFile(env, fname, "line one\n\n\nline two\nline three")); 101f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 102f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur for (auto buf_size : BufferSizes()) { 1036fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan std::unique_ptr<RandomAccessFile> file; 104f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 105f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string line; 1066fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan io::InputBuffer in(file.get(), buf_size); 107f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 108f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, "line one"); 109f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 110f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, ""); 111f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 112f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, ""); 113f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 114f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, "line two"); 115f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadLine(&line)); 116f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(line, "line three"); 117f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.ReadLine(&line))); 118f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur // A second call should also return end of file 119f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.ReadLine(&line))); 120f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur } 121f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur} 122f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 1231c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlowerTEST(InputBuffer, ReadLine_CRLF) { 1241c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower Env* env = Env::Default(); 1251c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower string fname = testing::TmpDir() + "/inputbuffer_test"; 126bc225bfaa534acc25047fe844f19edc333b7a76aPeter Hawkins TF_ASSERT_OK(WriteStringToFile(env, fname, 127bc225bfaa534acc25047fe844f19edc333b7a76aPeter Hawkins "line one\r\n\r\n\r\nline two\r\nline three")); 1281c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower 1291c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower for (auto buf_size : BufferSizes()) { 1306fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan std::unique_ptr<RandomAccessFile> file; 1311c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 1321c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower string line; 1336fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan io::InputBuffer in(file.get(), buf_size); 1341c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower TF_CHECK_OK(in.ReadLine(&line)); 1351c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower EXPECT_EQ(line, "line one"); 1361c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower TF_CHECK_OK(in.ReadLine(&line)); 1371c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower EXPECT_EQ(line, ""); 1381c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower TF_CHECK_OK(in.ReadLine(&line)); 1391c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower EXPECT_EQ(line, ""); 1401c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower TF_CHECK_OK(in.ReadLine(&line)); 1411c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower EXPECT_EQ(line, "line two"); 1421c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower TF_CHECK_OK(in.ReadLine(&line)); 1431c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower EXPECT_EQ(line, "line three"); 1441c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower EXPECT_TRUE(errors::IsOutOfRange(in.ReadLine(&line))); 1451c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower // A second call should also return end of file 1461c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower EXPECT_TRUE(errors::IsOutOfRange(in.ReadLine(&line))); 1471c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower } 1481c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower} 1491c579361cd1e088dd5e05a394b1561a73e3667baA. Unique TensorFlower 150f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath KudlurTEST(InputBuffer, ReadNBytes) { 151f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur Env* env = Env::Default(); 152f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string fname = testing::TmpDir() + "/inputbuffer_test"; 153bc225bfaa534acc25047fe844f19edc333b7a76aPeter Hawkins TF_ASSERT_OK(WriteStringToFile(env, fname, "0123456789")); 154f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 15594054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang // ReadNBytes(int64, string*). 156f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur for (auto buf_size : BufferSizes()) { 1576fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan std::unique_ptr<RandomAccessFile> file; 158f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 159f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string read; 1606fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan io::InputBuffer in(file.get(), buf_size); 161f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(0, in.Tell()); 162f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadNBytes(3, &read)); 163f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(read, "012"); 164f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(3, in.Tell()); 165f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadNBytes(0, &read)); 166f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(read, ""); 167f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(3, in.Tell()); 168f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadNBytes(4, &read)); 169f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(read, "3456"); 170f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(7, in.Tell()); 171f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadNBytes(0, &read)); 172f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(read, ""); 173f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(7, in.Tell()); 174f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.ReadNBytes(5, &read))); 175f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(read, "789"); 176f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(10, in.Tell()); 177f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.ReadNBytes(5, &read))); 178f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(read, ""); 179f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(10, in.Tell()); 180f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadNBytes(0, &read)); 181f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(read, ""); 182f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(10, in.Tell()); 183f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur } 18494054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang // ReadNBytes(int64, char*, size_t*). 18594054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang size_t bytes_read; 18694054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang for (auto buf_size : BufferSizes()) { 18794054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang std::unique_ptr<RandomAccessFile> file; 18894054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 18994054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang char read[5]; 19094054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang io::InputBuffer in(file.get(), buf_size); 19194054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 19294054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(0, in.Tell()); 19394054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_ASSERT_OK(in.ReadNBytes(3, read, &bytes_read)); 19494054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(StringPiece(read, 3), "012"); 19594054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 19694054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(3, in.Tell()); 19794054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_ASSERT_OK(in.ReadNBytes(0, read, &bytes_read)); 19894054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(StringPiece(read, 3), "012"); 19994054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 20094054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(3, in.Tell()); 20194054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_ASSERT_OK(in.ReadNBytes(4, read, &bytes_read)); 20294054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(StringPiece(read, 4), "3456"); 20394054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 20494054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(7, in.Tell()); 20594054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_ASSERT_OK(in.ReadNBytes(0, read, &bytes_read)); 20694054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(StringPiece(read, 4), "3456"); 20794054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 20894054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(7, in.Tell()); 20994054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_TRUE(errors::IsOutOfRange(in.ReadNBytes(5, read, &bytes_read))); 21094054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(StringPiece(read, 3), "789"); 21194054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 21294054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(10, in.Tell()); 21394054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_TRUE(errors::IsOutOfRange(in.ReadNBytes(5, read, &bytes_read))); 21494054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(StringPiece(read, 3), "789"); 21594054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 21694054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(10, in.Tell()); 21794054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_ASSERT_OK(in.ReadNBytes(0, read, &bytes_read)); 21894054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(StringPiece(read, 3), "789"); 21994054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(10, in.Tell()); 22094054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang } 221f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur} 222f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 223f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath KudlurTEST(InputBuffer, SkipNBytes) { 224f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur Env* env = Env::Default(); 225f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string fname = testing::TmpDir() + "/inputbuffer_test"; 226bc225bfaa534acc25047fe844f19edc333b7a76aPeter Hawkins TF_ASSERT_OK(WriteStringToFile(env, fname, "0123456789")); 227f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 228f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur for (auto buf_size : BufferSizes()) { 2296fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan std::unique_ptr<RandomAccessFile> file; 230f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 231f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur string read; 2326fb0fe23a6cc015c7f37f53559abf5172687366eVijay Vasudevan io::InputBuffer in(file.get(), buf_size); 233f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(0, in.Tell()); 234f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.SkipNBytes(3)); 235f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(3, in.Tell()); 236f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.SkipNBytes(0)); 237f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(3, in.Tell()); 238f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadNBytes(2, &read)); 239f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(read, "34"); 240f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(5, in.Tell()); 241f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.SkipNBytes(0)); 242f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(5, in.Tell()); 243f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.SkipNBytes(2)); 244f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(7, in.Tell()); 245f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur TF_CHECK_OK(in.ReadNBytes(1, &read)); 246f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(read, "7"); 247f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(8, in.Tell()); 248f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.SkipNBytes(5))); 249f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(10, in.Tell()); 250f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.SkipNBytes(5))); 251f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(10, in.Tell()); 252f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_TRUE(errors::IsOutOfRange(in.ReadNBytes(5, &read))); 253f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(read, ""); 254f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur EXPECT_EQ(10, in.Tell()); 255f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur } 256f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur} 257f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur 25894054e083e03b2e7a298a0efbee560746f04d470Zongheng YangTEST(InputBuffer, Seek) { 25994054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang Env* env = Env::Default(); 26094054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang string fname = testing::TmpDir() + "/inputbuffer_test"; 261bc225bfaa534acc25047fe844f19edc333b7a76aPeter Hawkins TF_ASSERT_OK(WriteStringToFile(env, fname, "0123456789")); 26294054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 26394054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang for (auto buf_size : BufferSizes()) { 26494054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang std::unique_ptr<RandomAccessFile> file; 26594054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 26694054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang string read; 26794054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang io::InputBuffer in(file.get(), buf_size); 26894054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 26994054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(in.ReadNBytes(3, &read)); 27094054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(read, "012"); 27194054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(in.ReadNBytes(3, &read)); 27294054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(read, "345"); 27394054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 27494054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(in.Seek(0)); 27594054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(in.ReadNBytes(3, &read)); 27694054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(read, "012"); 27794054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 27894054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(in.Seek(3)); 27994054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(in.ReadNBytes(4, &read)); 28094054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(read, "3456"); 28194054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 28294054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(in.Seek(4)); 28394054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(in.ReadNBytes(4, &read)); 28494054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_EQ(read, "4567"); 28594054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 28694054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang TF_CHECK_OK(in.Seek(1 << 25)); 28794054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_TRUE(errors::IsOutOfRange(in.ReadNBytes(1, &read))); 28894054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 28994054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang EXPECT_TRUE( 29094054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang StringPiece(in.Seek(-1).ToString()).contains("negative position")); 29194054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang } 29294054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang} 29394054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang 2942f3328b03f2ef261781b02ca3571688356a7915cZongheng YangTEST(InputBuffer, ReadVarint32) { 2952f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang Env* env = Env::Default(); 2962f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang string fname = testing::TmpDir() + "/inputbuffer_test"; 2972f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang 2982f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang // Generates data. 2992f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang std::vector<uint32> data; 3002f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang uint32 i = 0; 30148a4853ebeabfa32b360e8505caf4ec31a45cdb3A. Unique TensorFlower for (; i < (1U << 10); i += 1) data.push_back(i); 30248a4853ebeabfa32b360e8505caf4ec31a45cdb3A. Unique TensorFlower for (; i < (1U << 15); i += 5) data.push_back(i); 30348a4853ebeabfa32b360e8505caf4ec31a45cdb3A. Unique TensorFlower for (; i < (1U << 31); i += 132817) data.push_back(i); 3042f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang data.push_back(std::numeric_limits<uint32>::max()); 3052f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang 3062f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang // Writes the varints. 3072f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang { 3082f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang std::unique_ptr<WritableFile> file; 3092f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang TF_CHECK_OK(env->NewWritableFile(fname, &file)); 3102f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang string varint; 3112f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang for (uint32 number : data) { 3122f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang varint.clear(); 3132f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang core::PutVarint32(&varint, number); 3142f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang TF_CHECK_OK(file->Append(StringPiece(varint))); 3152f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang } 3162f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang } 3172f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang 3182f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang for (auto buf_size : BufferSizes()) { 3192f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang std::unique_ptr<RandomAccessFile> file; 3202f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 3212f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang io::InputBuffer in(file.get(), buf_size); 3222f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang uint32 result = 0; 3232f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang 3242f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang for (uint32 expected : data) { 3252f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang TF_ASSERT_OK(in.ReadVarint32(&result)); 3262f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang EXPECT_EQ(expected, result); 3272f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang } 3282f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang EXPECT_TRUE(errors::IsOutOfRange(in.ReadVarint32(&result))); 3292f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang } 3302f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang} 3312f3328b03f2ef261781b02ca3571688356a7915cZongheng Yang 33224f9c6e0dbd449624aa1db543550ec412975492eSaurabh SaxenaTEST(InputBuffer, ReadVarint64) { 33324f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena Env* env = Env::Default(); 33424f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena string fname = testing::TmpDir() + "/inputbuffer_test"; 33524f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena 33624f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena // Generates data. 33724f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena std::vector<uint64> data; 33824f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena uint64 i = 0; 33924f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena for (; i < (1U << 10); i += 1) data.push_back(i); 34024f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena for (; i < (1U << 15); i += 5) data.push_back(i); 34124f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena for (; i < (1U << 31); i += 164817) data.push_back(i); 34224f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena for (; i < (1ULL << 63); i += 16481797854795663UL) data.push_back(i); 34324f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena data.push_back(std::numeric_limits<uint64>::max()); 34424f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena 34524f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena // Writes the varints. 34624f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena { 34724f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena std::unique_ptr<WritableFile> file; 34824f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena TF_CHECK_OK(env->NewWritableFile(fname, &file)); 34924f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena string varint; 35024f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena for (uint64 number : data) { 35124f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena varint.clear(); 35224f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena core::PutVarint64(&varint, number); 35324f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena TF_CHECK_OK(file->Append(StringPiece(varint))); 35424f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena } 35524f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena } 35624f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena 35724f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena for (auto buf_size : BufferSizes()) { 35824f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena std::unique_ptr<RandomAccessFile> file; 35924f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); 36024f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena io::InputBuffer in(file.get(), buf_size); 36124f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena uint64 result = 0; 36224f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena 36324f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena for (uint64 expected : data) { 36424f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena TF_ASSERT_OK(in.ReadVarint64(&result)); 36524f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena EXPECT_EQ(expected, result); 36624f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena } 36724f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena EXPECT_TRUE(errors::IsOutOfRange(in.ReadVarint64(&result))); 36824f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena } 36924f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena} 37024f9c6e0dbd449624aa1db543550ec412975492eSaurabh Saxena 37194054e083e03b2e7a298a0efbee560746f04d470Zongheng Yang} // namespace 372f41959ccb2d9d4c722fe8fc3351401d53bcf490Manjunath Kudlur} // namespace tensorflow 373