1027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe/* 2027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe * Copyright (C) 2017 The Android Open Source Project 3027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe * 4027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe * Licensed under the Apache License, Version 2.0 (the "License"); 5027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe * you may not use this file except in compliance with the License. 6027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe * You may obtain a copy of the License at 7027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe * 8027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe * http://www.apache.org/licenses/LICENSE-2.0 9027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe * 10027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe * Unless required by applicable law or agreed to in writing, software 11027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe * distributed under the License is distributed on an "AS IS" BASIS, 12027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe * See the License for the specific language governing permissions and 14027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe * limitations under the License. 15027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe */ 16027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 17027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe#ifndef ART_TEST_TI_AGENT_TI_UTF_H_ 18027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe#define ART_TEST_TI_AGENT_TI_UTF_H_ 19027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 20027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe#include <inttypes.h> 21027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe#include <string.h> 22027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 23027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe#include "android-base/logging.h" 24027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 25027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampenamespace art { 26027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampenamespace ti { 27027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 28027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampeinline size_t CountModifiedUtf8Chars(const char* utf8, size_t byte_count) { 29027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe DCHECK_LE(byte_count, strlen(utf8)); 30027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe size_t len = 0; 31027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe const char* end = utf8 + byte_count; 32027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe for (; utf8 < end; ++utf8) { 33027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe int ic = *utf8; 34027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe len++; 35027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe if (LIKELY((ic & 0x80) == 0)) { 36027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // One-byte encoding. 37027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe continue; 38027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } 39027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // Two- or three-byte encoding. 40027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe utf8++; 41027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe if ((ic & 0x20) == 0) { 42027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // Two-byte encoding. 43027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe continue; 44027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } 45027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe utf8++; 46027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe if ((ic & 0x10) == 0) { 47027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // Three-byte encoding. 48027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe continue; 49027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } 50027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 51027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // Four-byte encoding: needs to be converted into a surrogate 52027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // pair. 53027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe utf8++; 54027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe len++; 55027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } 56027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe return len; 57027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe} 58027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 59027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampeinline uint16_t GetTrailingUtf16Char(uint32_t maybe_pair) { 60027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe return static_cast<uint16_t>(maybe_pair >> 16); 61027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe} 62027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 63027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampeinline uint16_t GetLeadingUtf16Char(uint32_t maybe_pair) { 64027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe return static_cast<uint16_t>(maybe_pair & 0x0000FFFF); 65027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe} 66027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 67027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampeinline uint32_t GetUtf16FromUtf8(const char** utf8_data_in) { 68027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe const uint8_t one = *(*utf8_data_in)++; 69027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe if ((one & 0x80) == 0) { 70027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // one-byte encoding 71027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe return one; 72027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } 73027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 74027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe const uint8_t two = *(*utf8_data_in)++; 75027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe if ((one & 0x20) == 0) { 76027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // two-byte encoding 77027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe return ((one & 0x1f) << 6) | (two & 0x3f); 78027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } 79027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 80027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe const uint8_t three = *(*utf8_data_in)++; 81027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe if ((one & 0x10) == 0) { 82027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe return ((one & 0x0f) << 12) | ((two & 0x3f) << 6) | (three & 0x3f); 83027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } 84027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 85027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // Four byte encodings need special handling. We'll have 86027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // to convert them into a surrogate pair. 87027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe const uint8_t four = *(*utf8_data_in)++; 88027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 89027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // Since this is a 4 byte UTF-8 sequence, it will lie between 90027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // U+10000 and U+1FFFFF. 91027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // 92027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // TODO: What do we do about values in (U+10FFFF, U+1FFFFF) ? The 93027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // spec says they're invalid but nobody appears to check for them. 94027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe const uint32_t code_point = ((one & 0x0f) << 18) | ((two & 0x3f) << 12) 95027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe | ((three & 0x3f) << 6) | (four & 0x3f); 96027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 97027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe uint32_t surrogate_pair = 0; 98027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // Step two: Write out the high (leading) surrogate to the bottom 16 bits 99027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // of the of the 32 bit type. 100027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe surrogate_pair |= ((code_point >> 10) + 0xd7c0) & 0xffff; 101027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // Step three : Write out the low (trailing) surrogate to the top 16 bits. 102027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe surrogate_pair |= ((code_point & 0x03ff) + 0xdc00) << 16; 103027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 104027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe return surrogate_pair; 105027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe} 106027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 107027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampeinline void ConvertUtf16ToModifiedUtf8(char* utf8_out, 108027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe size_t byte_count, 109027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe const uint16_t* utf16_in, 110027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe size_t char_count) { 111027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe if (LIKELY(byte_count == char_count)) { 112027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // Common case where all characters are ASCII. 113027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe const uint16_t *utf16_end = utf16_in + char_count; 114027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe for (const uint16_t *p = utf16_in; p < utf16_end;) { 115027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe *utf8_out++ = static_cast<char>(*p++); 116027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } 117027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe return; 118027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } 119027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 120027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // String contains non-ASCII characters. 121027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe while (char_count--) { 122027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe const uint16_t ch = *utf16_in++; 123027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe if (ch > 0 && ch <= 0x7f) { 124027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe *utf8_out++ = ch; 125027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } else { 126027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // Char_count == 0 here implies we've encountered an unpaired 127027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // surrogate and we have no choice but to encode it as 3-byte UTF 128027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // sequence. Note that unpaired surrogates can occur as a part of 129027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // "normal" operation. 130027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe if ((ch >= 0xd800 && ch <= 0xdbff) && (char_count > 0)) { 131027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe const uint16_t ch2 = *utf16_in; 132027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 133027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // Check if the other half of the pair is within the expected 134027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // range. If it isn't, we will have to emit both "halves" as 135027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // separate 3 byte sequences. 136027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe if (ch2 >= 0xdc00 && ch2 <= 0xdfff) { 137027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe utf16_in++; 138027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe char_count--; 139027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe const uint32_t code_point = (ch << 10) + ch2 - 0x035fdc00; 140027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe *utf8_out++ = (code_point >> 18) | 0xf0; 141027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe *utf8_out++ = ((code_point >> 12) & 0x3f) | 0x80; 142027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe *utf8_out++ = ((code_point >> 6) & 0x3f) | 0x80; 143027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe *utf8_out++ = (code_point & 0x3f) | 0x80; 144027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe continue; 145027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } 146027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } 147027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 148027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe if (ch > 0x07ff) { 149027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // Three byte encoding. 150027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe *utf8_out++ = (ch >> 12) | 0xe0; 151027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe *utf8_out++ = ((ch >> 6) & 0x3f) | 0x80; 152027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe *utf8_out++ = (ch & 0x3f) | 0x80; 153027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } else /*(ch > 0x7f || ch == 0)*/ { 154027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // Two byte encoding. 155027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe *utf8_out++ = (ch >> 6) | 0xc0; 156027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe *utf8_out++ = (ch & 0x3f) | 0x80; 157027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } 158027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } 159027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } 160027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe} 161027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 162027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampeinline size_t CountUtf8Bytes(const uint16_t* chars, size_t char_count) { 163027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe size_t result = 0; 164027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe const uint16_t *end = chars + char_count; 165027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe while (chars < end) { 166027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe const uint16_t ch = *chars++; 167027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe if (LIKELY(ch != 0 && ch < 0x80)) { 168027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe result++; 169027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe continue; 170027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } 171027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe if (ch < 0x800) { 172027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe result += 2; 173027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe continue; 174027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } 175027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe if (ch >= 0xd800 && ch < 0xdc00) { 176027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe if (chars < end) { 177027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe const uint16_t ch2 = *chars; 178027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // If we find a properly paired surrogate, we emit it as a 4 byte 179027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // UTF sequence. If we find an unpaired leading or trailing surrogate, 180027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe // we emit it as a 3 byte sequence like would have done earlier. 181027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe if (ch2 >= 0xdc00 && ch2 < 0xe000) { 182027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe chars++; 183027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe result += 4; 184027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe continue; 185027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } 186027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } 187027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } 188027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe result += 3; 189027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe } 190027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe return result; 191027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe} 192027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 193027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe} // namespace ti 194027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe} // namespace art 195027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe 196027444b64dd52e1d2beea7aa525fbb8146a516bcAndreas Gampe#endif // ART_TEST_TI_AGENT_TI_UTF_H_ 197