1ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen// Copyright (c) 2011 The Chromium Authors. All rights reserved. 2c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// Use of this source code is governed by a BSD-style license that can be 3c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// found in the LICENSE file. 4c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 5ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen#include <algorithm> 6ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen 7c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#include "base/logging.h" 8c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#include "base/string_piece.h" 9c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#include "base/utf_offset_string_conversions.h" 10c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#include "testing/gtest/include/gtest/gtest.h" 11c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 12c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottnamespace base { 13c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 14c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottnamespace { 15c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 16ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsenstatic const size_t kNpos = std::wstring::npos; 17ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen 18c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// Given a null-terminated string of wchar_t with each wchar_t representing 19c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// a UTF-16 code unit, returns a string16 made up of wchar_t's in the input. 20c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// Each wchar_t should be <= 0xFFFF and a non-BMP character (> U+FFFF) 21c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// should be represented as a surrogate pair (two UTF-16 units) 22c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// *even* where wchar_t is 32-bit (Linux and Mac). 23c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// 24c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// This is to help write tests for functions with string16 params until 25c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// the C++ 0x UTF-16 literal is well-supported by compilers. 26c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottstring16 BuildString16(const wchar_t* s) { 27c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#if defined(WCHAR_T_IS_UTF16) 28c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return string16(s); 29c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#elif defined(WCHAR_T_IS_UTF32) 30c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott string16 u16; 31c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott while (*s != 0) { 32c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott DCHECK(static_cast<unsigned int>(*s) <= 0xFFFFu); 33c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott u16.push_back(*s++); 34c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott } 35c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return u16; 36c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#endif 37c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 38c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 39c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} // namespace 40c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 41c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick ScottTEST(UTFOffsetStringConversionsTest, AdjustOffset) { 42c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott struct UTF8ToWideCase { 43c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott const char* utf8; 44c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott size_t input_offset; 45c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott size_t output_offset; 46c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott } utf8_to_wide_cases[] = { 47ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen {"", 0, kNpos}, 48ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen {"\xe4\xbd\xa0\xe5\xa5\xbd", 1, kNpos}, 49c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott {"\xe4\xbd\xa0\xe5\xa5\xbd", 3, 1}, 50c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott {"\xed\xb0\x80z", 3, 1}, 51c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott {"A\xF0\x90\x8C\x80z", 1, 1}, 52ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen {"A\xF0\x90\x8C\x80z", 2, kNpos}, 53c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#if defined(WCHAR_T_IS_UTF16) 54c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott {"A\xF0\x90\x8C\x80z", 5, 3}, 55c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#elif defined(WCHAR_T_IS_UTF32) 56c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott {"A\xF0\x90\x8C\x80z", 5, 2}, 57c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#endif 58c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott }; 59c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott for (size_t i = 0; i < ARRAYSIZE_UNSAFE(utf8_to_wide_cases); ++i) { 60c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott size_t offset = utf8_to_wide_cases[i].input_offset; 61c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott UTF8ToWideAndAdjustOffset(utf8_to_wide_cases[i].utf8, &offset); 62c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott EXPECT_EQ(utf8_to_wide_cases[i].output_offset, offset); 63c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott } 64c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 65c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#if defined(WCHAR_T_IS_UTF32) 66c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott struct UTF16ToWideCase { 67c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott const wchar_t* wide; 68c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott size_t input_offset; 69c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott size_t output_offset; 70c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott } utf16_to_wide_cases[] = { 71c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott {L"\xD840\xDC00\x4E00", 0, 0}, 72ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen {L"\xD840\xDC00\x4E00", 1, kNpos}, 73c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott {L"\xD840\xDC00\x4E00", 2, 1}, 74c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott }; 75c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott for (size_t i = 0; i < ARRAYSIZE_UNSAFE(utf16_to_wide_cases); ++i) { 76c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott size_t offset = utf16_to_wide_cases[i].input_offset; 77c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott UTF16ToWideAndAdjustOffset(BuildString16(utf16_to_wide_cases[i].wide), 78c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott &offset); 79c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott EXPECT_EQ(utf16_to_wide_cases[i].output_offset, offset); 80c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott } 81c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#endif 82c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 83c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 84ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian MonsenTEST(UTFOffsetStringConversionsTest, LimitOffsets) { 85ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen const size_t kLimit = 10; 86ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen const size_t kItems = 20; 87ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen std::vector<size_t> size_ts; 88ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen for (size_t t = 0; t < kItems; ++t) 89ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen size_ts.push_back(t); 90ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen std::for_each(size_ts.begin(), size_ts.end(), 91ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen LimitOffset<std::wstring>(kLimit)); 92ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen size_t unlimited_count = 0; 93ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen for (std::vector<size_t>::iterator ti = size_ts.begin(); ti != size_ts.end(); 94ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen ++ti) { 95ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen if (*ti < kLimit && *ti != kNpos) 96ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen ++unlimited_count; 97ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen } 98ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen EXPECT_EQ(10U, unlimited_count); 99ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen 100ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen // Reverse the values in the vector and try again. 101ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen size_ts.clear(); 102ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen for (size_t t = kItems; t > 0; --t) 103ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen size_ts.push_back(t - 1); 104ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen std::for_each(size_ts.begin(), size_ts.end(), 105ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen LimitOffset<std::wstring>(kLimit)); 106ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen unlimited_count = 0; 107ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen for (std::vector<size_t>::iterator ti = size_ts.begin(); ti != size_ts.end(); 108ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen ++ti) { 109ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen if (*ti < kLimit && *ti != kNpos) 110ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen ++unlimited_count; 111ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen } 112ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen EXPECT_EQ(10U, unlimited_count); 113ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen} 114ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen 115ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian MonsenTEST(UTFOffsetStringConversionsTest, AdjustOffsets) { 116ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen // Imagine we have strings as shown in the following cases where the 117ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen // X's represent encoded characters. 118ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen // 1: abcXXXdef ==> abcXdef 119ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen std::vector<size_t> offsets; 120ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen for (size_t t = 0; t < 9; ++t) 121ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen offsets.push_back(t); 122ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen AdjustOffset::Adjustments adjustments; 123ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen adjustments.push_back(AdjustOffset::Adjustment(3, 3, 1)); 124ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen std::for_each(offsets.begin(), offsets.end(), AdjustOffset(adjustments)); 125ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen size_t expected_1[] = {0, 1, 2, 3, kNpos, kNpos, 4, 5, 6}; 126ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen EXPECT_EQ(offsets.size(), arraysize(expected_1)); 127ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen for (size_t i = 0; i < arraysize(expected_1); ++i) 128ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen EXPECT_EQ(expected_1[i], offsets[i]); 129ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen 130ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen // 2: XXXaXXXXbcXXXXXXXdefXXX ==> XaXXbcXXXXdefX 131ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen offsets.clear(); 132ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen for (size_t t = 0; t < 23; ++t) 133ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen offsets.push_back(t); 134ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen adjustments.clear(); 135ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen adjustments.push_back(AdjustOffset::Adjustment(0, 3, 1)); 136ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen adjustments.push_back(AdjustOffset::Adjustment(4, 4, 2)); 137ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen adjustments.push_back(AdjustOffset::Adjustment(10, 7, 4)); 138ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen adjustments.push_back(AdjustOffset::Adjustment(20, 3, 1)); 139ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen std::for_each(offsets.begin(), offsets.end(), AdjustOffset(adjustments)); 140ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen size_t expected_2[] = {0, kNpos, kNpos, 1, 2, kNpos, kNpos, kNpos, 4, 5, 6, 141ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen kNpos, kNpos, kNpos, kNpos, kNpos, kNpos, 10, 11, 12, 142ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen 13, kNpos, kNpos}; 143ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen EXPECT_EQ(offsets.size(), arraysize(expected_2)); 144ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen for (size_t i = 0; i < arraysize(expected_2); ++i) 145ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen EXPECT_EQ(expected_2[i], offsets[i]); 146ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen 147ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen // 3: XXXaXXXXbcdXXXeXX ==> aXXXXbcdXXXe 148ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen offsets.clear(); 149ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen for (size_t t = 0; t < 17; ++t) 150ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen offsets.push_back(t); 151ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen adjustments.clear(); 152ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen adjustments.push_back(AdjustOffset::Adjustment(0, 3, 0)); 153ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen adjustments.push_back(AdjustOffset::Adjustment(4, 4, 4)); 154ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen adjustments.push_back(AdjustOffset::Adjustment(11, 3, 3)); 155ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen adjustments.push_back(AdjustOffset::Adjustment(15, 2, 0)); 156ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen std::for_each(offsets.begin(), offsets.end(), AdjustOffset(adjustments)); 157ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen size_t expected_3[] = {kNpos, kNpos, kNpos, 0, 1, kNpos, kNpos, kNpos, 5, 6, 158ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen 7, 8, kNpos, kNpos, 11, kNpos, kNpos}; 159ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen EXPECT_EQ(offsets.size(), arraysize(expected_3)); 160ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen for (size_t i = 0; i < arraysize(expected_3); ++i) 161ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen EXPECT_EQ(expected_3[i], offsets[i]); 162ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen} 163ddb351dbec246cf1fab5ec20d2d5520909041de1Kristian Monsen 164c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} // namaspace base 165