1ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root/* 2ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * Copyright (C) 2005 The Android Open Source Project 3ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * 4ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * Licensed under the Apache License, Version 2.0 (the "License"); 5ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * you may not use this file except in compliance with the License. 6ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * You may obtain a copy of the License at 7ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * 8ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * http://www.apache.org/licenses/LICENSE-2.0 9ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * 10ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * Unless required by applicable law or agreed to in writing, software 11ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * distributed under the License is distributed on an "AS IS" BASIS, 12ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * See the License for the specific language governing permissions and 14ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * limitations under the License. 15ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root */ 16ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 17ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root#include <utils/Unicode.h> 18ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 19ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root#include <stddef.h> 20ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 21ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root#ifdef HAVE_WINSOCK 22ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root# undef nhtol 23ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root# undef htonl 24ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root# undef nhtos 25ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root# undef htons 26ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 27ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root# ifdef HAVE_LITTLE_ENDIAN 28ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root# define ntohl(x) ( ((x) << 24) | (((x) >> 24) & 255) | (((x) << 8) & 0xff0000) | (((x) >> 8) & 0xff00) ) 29ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root# define htonl(x) ntohl(x) 30ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root# define ntohs(x) ( (((x) << 8) & 0xff00) | (((x) >> 8) & 255) ) 31ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root# define htons(x) ntohs(x) 32ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root# else 33ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root# define ntohl(x) (x) 34ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root# define htonl(x) (x) 35ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root# define ntohs(x) (x) 36ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root# define htons(x) (x) 37ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root# endif 38ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root#else 39ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root# include <netinet/in.h> 40ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root#endif 41ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 42ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootextern "C" { 43ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 44ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootstatic const char32_t kByteMask = 0x000000BF; 45ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootstatic const char32_t kByteMark = 0x00000080; 46ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 47ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// Surrogates aren't valid for UTF-32 characters, so define some 48ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// constants that will let us screen them out. 49ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootstatic const char32_t kUnicodeSurrogateHighStart = 0x0000D800; 50ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootstatic const char32_t kUnicodeSurrogateHighEnd = 0x0000DBFF; 51ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootstatic const char32_t kUnicodeSurrogateLowStart = 0x0000DC00; 52ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootstatic const char32_t kUnicodeSurrogateLowEnd = 0x0000DFFF; 53ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootstatic const char32_t kUnicodeSurrogateStart = kUnicodeSurrogateHighStart; 54ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootstatic const char32_t kUnicodeSurrogateEnd = kUnicodeSurrogateLowEnd; 55ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootstatic const char32_t kUnicodeMaxCodepoint = 0x0010FFFF; 56ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 57ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// Mask used to set appropriate bits in first byte of UTF-8 sequence, 58ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// indexed by number of bytes in the sequence. 59ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// 0xxxxxxx 60ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// -> (00-7f) 7bit. Bit mask for the first byte is 0x00000000 61ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// 110yyyyx 10xxxxxx 62ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// -> (c0-df)(80-bf) 11bit. Bit mask is 0x000000C0 63ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// 1110yyyy 10yxxxxx 10xxxxxx 64ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// -> (e0-ef)(80-bf)(80-bf) 16bit. Bit mask is 0x000000E0 65ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// 11110yyy 10yyxxxx 10xxxxxx 10xxxxxx 66ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// -> (f0-f7)(80-bf)(80-bf)(80-bf) 21bit. Bit mask is 0x000000F0 67ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootstatic const char32_t kFirstByteMark[] = { 68ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 0x00000000, 0x00000000, 0x000000C0, 0x000000E0, 0x000000F0 69ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root}; 70ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 71ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// -------------------------------------------------------------------------- 72ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// UTF-32 73ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// -------------------------------------------------------------------------- 74ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 75ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root/** 76ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * Return number of UTF-8 bytes required for the character. If the character 77ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * is invalid, return size of 0. 78ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root */ 79ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootstatic inline size_t utf32_codepoint_utf8_length(char32_t srcChar) 80ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 81ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root // Figure out how many bytes the result will require. 82ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if (srcChar < 0x00000080) { 83ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return 1; 84ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } else if (srcChar < 0x00000800) { 85ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return 2; 86ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } else if (srcChar < 0x00010000) { 87ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if ((srcChar < kUnicodeSurrogateStart) || (srcChar > kUnicodeSurrogateEnd)) { 88ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return 3; 89ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } else { 90ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root // Surrogates are invalid UTF-32 characters. 91ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return 0; 92ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 93ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 94ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root // Max code point for Unicode is 0x0010FFFF. 95ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root else if (srcChar <= kUnicodeMaxCodepoint) { 96ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return 4; 97ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } else { 98ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root // Invalid UTF-32 character. 99ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return 0; 100ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 101ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 102ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 103ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// Write out the source character to <dstP>. 104ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 105ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootstatic inline void utf32_codepoint_to_utf8(uint8_t* dstP, char32_t srcChar, size_t bytes) 106ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 107ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root dstP += bytes; 108ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root switch (bytes) 109ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root { /* note: everything falls through. */ 110ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root case 4: *--dstP = (uint8_t)((srcChar | kByteMark) & kByteMask); srcChar >>= 6; 111ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root case 3: *--dstP = (uint8_t)((srcChar | kByteMark) & kByteMask); srcChar >>= 6; 112ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root case 2: *--dstP = (uint8_t)((srcChar | kByteMark) & kByteMask); srcChar >>= 6; 113ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root case 1: *--dstP = (uint8_t)(srcChar | kFirstByteMark[bytes]); 114ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 115ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 116ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 117ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootsize_t strlen32(const char32_t *s) 118ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 119ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char32_t *ss = s; 120ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root while ( *ss ) 121ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root ss++; 122ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return ss-s; 123ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 124ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 125ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootsize_t strnlen32(const char32_t *s, size_t maxlen) 126ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 127ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char32_t *ss = s; 128ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root while ((maxlen > 0) && *ss) { 129ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root ss++; 130ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root maxlen--; 131ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 132ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return ss-s; 133ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 134ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 135ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootstatic inline int32_t utf32_at_internal(const char* cur, size_t *num_read) 136ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 137ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char first_char = *cur; 138ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if ((first_char & 0x80) == 0) { // ASCII 139ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root *num_read = 1; 140ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return *cur; 141ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 142ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root cur++; 143ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root char32_t mask, to_ignore_mask; 144ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root size_t num_to_read = 0; 145ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root char32_t utf32 = first_char; 146ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root for (num_to_read = 1, mask = 0x40, to_ignore_mask = 0xFFFFFF80; 147ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root (first_char & mask); 148ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root num_to_read++, to_ignore_mask |= mask, mask >>= 1) { 149ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root // 0x3F == 00111111 150ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root utf32 = (utf32 << 6) + (*cur++ & 0x3F); 151ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 152ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root to_ignore_mask |= mask; 153ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root utf32 &= ~(to_ignore_mask << (6 * (num_to_read - 1))); 154ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 155ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root *num_read = num_to_read; 156ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return static_cast<int32_t>(utf32); 157ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 158ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 159ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootint32_t utf32_from_utf8_at(const char *src, size_t src_len, size_t index, size_t *next_index) 160ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 161ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if (index >= src_len) { 162ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return -1; 163ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 164ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root size_t dummy_index; 165ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if (next_index == NULL) { 166ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root next_index = &dummy_index; 167ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 168ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root size_t num_read; 169ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root int32_t ret = utf32_at_internal(src + index, &num_read); 170ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if (ret >= 0) { 171ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root *next_index = index + num_read; 172ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 173ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 174ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return ret; 175ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 176ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 177ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootssize_t utf32_to_utf8_length(const char32_t *src, size_t src_len) 178ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 179ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if (src == NULL || src_len == 0) { 180ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return -1; 181ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 182ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 183ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root size_t ret = 0; 184ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char32_t *end = src + src_len; 185ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root while (src < end) { 186ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root ret += utf32_codepoint_utf8_length(*src++); 187ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 188ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return ret; 189ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 190ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 191ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootvoid utf32_to_utf8(const char32_t* src, size_t src_len, char* dst) 192ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 193ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if (src == NULL || src_len == 0 || dst == NULL) { 194ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return; 195ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 196ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 197ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char32_t *cur_utf32 = src; 198ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char32_t *end_utf32 = src + src_len; 199ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root char *cur = dst; 200ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root while (cur_utf32 < end_utf32) { 201ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root size_t len = utf32_codepoint_utf8_length(*cur_utf32); 202ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root utf32_codepoint_to_utf8((uint8_t *)cur, *cur_utf32++, len); 203ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root cur += len; 204ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 205ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root *cur = '\0'; 206ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 207ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 208ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// -------------------------------------------------------------------------- 209ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// UTF-16 210ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// -------------------------------------------------------------------------- 211ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 212ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootint strcmp16(const char16_t *s1, const char16_t *s2) 213ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 214ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root char16_t ch; 215ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root int d = 0; 216ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 217ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root while ( 1 ) { 218ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root d = (int)(ch = *s1++) - (int)*s2++; 219ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if ( d || !ch ) 220ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root break; 221ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 222ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 223ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return d; 224ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 225ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 226ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootint strncmp16(const char16_t *s1, const char16_t *s2, size_t n) 227ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 228ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root char16_t ch; 229ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root int d = 0; 230ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 231ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root while ( n-- ) { 232ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root d = (int)(ch = *s1++) - (int)*s2++; 233ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if ( d || !ch ) 234ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root break; 235ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 236ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 237ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return d; 238ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 239ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 240ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootchar16_t *strcpy16(char16_t *dst, const char16_t *src) 241ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 242ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root char16_t *q = dst; 243ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char16_t *p = src; 244ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root char16_t ch; 245ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 246ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root do { 247ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root *q++ = ch = *p++; 248ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } while ( ch ); 249ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 250ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return dst; 251ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 252ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 253ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootsize_t strlen16(const char16_t *s) 254ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 255ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char16_t *ss = s; 256ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root while ( *ss ) 257ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root ss++; 258ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return ss-s; 259ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 260ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 261ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 262ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootchar16_t *strncpy16(char16_t *dst, const char16_t *src, size_t n) 263ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 264ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root char16_t *q = dst; 265ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char16_t *p = src; 266ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root char ch; 267ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 268ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root while (n) { 269ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root n--; 270ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root *q++ = ch = *p++; 271ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if ( !ch ) 272ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root break; 273ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 274ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 275ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root *q = 0; 276ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 277ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return dst; 278ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 279ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 280ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootsize_t strnlen16(const char16_t *s, size_t maxlen) 281ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 282ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char16_t *ss = s; 283ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 284ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root /* Important: the maxlen test must precede the reference through ss; 285ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root since the byte beyond the maximum may segfault */ 286ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root while ((maxlen > 0) && *ss) { 287ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root ss++; 288ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root maxlen--; 289ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 290ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return ss-s; 291ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 292ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 293ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootint strzcmp16(const char16_t *s1, size_t n1, const char16_t *s2, size_t n2) 294ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 295ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char16_t* e1 = s1+n1; 296ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char16_t* e2 = s2+n2; 297ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 298ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root while (s1 < e1 && s2 < e2) { 299ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const int d = (int)*s1++ - (int)*s2++; 300ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if (d) { 301ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return d; 302ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 303ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 304ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 305ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return n1 < n2 306ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root ? (0 - (int)*s2) 307ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root : (n1 > n2 308ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root ? ((int)*s1 - 0) 309ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root : 0); 310ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 311ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 312ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootint strzcmp16_h_n(const char16_t *s1H, size_t n1, const char16_t *s2N, size_t n2) 313ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 314ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char16_t* e1 = s1H+n1; 315ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char16_t* e2 = s2N+n2; 316ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 317ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root while (s1H < e1 && s2N < e2) { 318ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char16_t c2 = ntohs(*s2N); 319ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const int d = (int)*s1H++ - (int)c2; 320ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root s2N++; 321ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if (d) { 322ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return d; 323ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 324ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 325ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 326ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return n1 < n2 327ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root ? (0 - (int)ntohs(*s2N)) 328ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root : (n1 > n2 329ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root ? ((int)*s1H - 0) 330ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root : 0); 331ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 332ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 333ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootvoid utf16_to_utf8(const char16_t* src, size_t src_len, char* dst) 334ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 335ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if (src == NULL || src_len == 0 || dst == NULL) { 336ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return; 337ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 338ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 339ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char16_t* cur_utf16 = src; 340ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char16_t* const end_utf16 = src + src_len; 341ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root char *cur = dst; 342ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root while (cur_utf16 < end_utf16) { 343ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root char32_t utf32; 344ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root // surrogate pairs 34572299bf0d240072174f847d13f1c9498b3ef9fa6Cylen Yao if((*cur_utf16 & 0xFC00) == 0xD800 && (cur_utf16 + 1) < end_utf16 34672299bf0d240072174f847d13f1c9498b3ef9fa6Cylen Yao && (*(cur_utf16 + 1) & 0xFC00) == 0xDC00) { 347ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root utf32 = (*cur_utf16++ - 0xD800) << 10; 348ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root utf32 |= *cur_utf16++ - 0xDC00; 349ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root utf32 += 0x10000; 350ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } else { 351ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root utf32 = (char32_t) *cur_utf16++; 352ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 353ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const size_t len = utf32_codepoint_utf8_length(utf32); 354ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root utf32_codepoint_to_utf8((uint8_t*)cur, utf32, len); 355ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root cur += len; 356ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 357ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root *cur = '\0'; 358ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 359ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 360ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// -------------------------------------------------------------------------- 361ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// UTF-8 362ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root// -------------------------------------------------------------------------- 363ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 364ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootssize_t utf8_length(const char *src) 365ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 366ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char *cur = src; 367ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root size_t ret = 0; 368ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root while (*cur != '\0') { 369ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char first_char = *cur++; 370ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if ((first_char & 0x80) == 0) { // ASCII 371ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root ret += 1; 372ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root continue; 373ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 374ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root // (UTF-8's character must not be like 10xxxxxx, 375ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root // but 110xxxxx, 1110xxxx, ... or 1111110x) 376ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if ((first_char & 0x40) == 0) { 377ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return -1; 378ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 379ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 380ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root int32_t mask, to_ignore_mask; 381ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root size_t num_to_read = 0; 382ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root char32_t utf32 = 0; 383ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root for (num_to_read = 1, mask = 0x40, to_ignore_mask = 0x80; 384ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root num_to_read < 5 && (first_char & mask); 385ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root num_to_read++, to_ignore_mask |= mask, mask >>= 1) { 386ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if ((*cur & 0xC0) != 0x80) { // must be 10xxxxxx 387ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return -1; 388ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 389ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root // 0x3F == 00111111 390ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root utf32 = (utf32 << 6) + (*cur++ & 0x3F); 391ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 392ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root // "first_char" must be (110xxxxx - 11110xxx) 393ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if (num_to_read == 5) { 394ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return -1; 395ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 396ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root to_ignore_mask |= mask; 397ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root utf32 |= ((~to_ignore_mask) & first_char) << (6 * (num_to_read - 1)); 398ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if (utf32 > kUnicodeMaxCodepoint) { 399ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return -1; 400ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 401ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 402ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root ret += num_to_read; 403ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 404ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return ret; 405ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 406ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 407ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootssize_t utf16_to_utf8_length(const char16_t *src, size_t src_len) 408ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 409ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if (src == NULL || src_len == 0) { 410ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return -1; 411ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 412ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 413ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root size_t ret = 0; 414ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char16_t* const end = src + src_len; 415ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root while (src < end) { 416ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if ((*src & 0xFC00) == 0xD800 && (src + 1) < end 417ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root && (*++src & 0xFC00) == 0xDC00) { 418ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root // surrogate pairs are always 4 bytes. 419ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root ret += 4; 420ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root src++; 421ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } else { 422ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root ret += utf32_codepoint_utf8_length((char32_t) *src++); 423ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 424ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 425ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return ret; 426ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 427ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 428ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root/** 429ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * Returns 1-4 based on the number of leading bits. 430ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * 431ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * 1111 -> 4 432ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * 1110 -> 3 433ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * 110x -> 2 434ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * 10xx -> 1 435ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * 0xxx -> 1 436ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root */ 437ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootstatic inline size_t utf8_codepoint_len(uint8_t ch) 438ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 439ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return ((0xe5000000 >> ((ch >> 3) & 0x1e)) & 3) + 1; 440ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 441ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 442ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootstatic inline void utf8_shift_and_mask(uint32_t* codePoint, const uint8_t byte) 443ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 444ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root *codePoint <<= 6; 445ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root *codePoint |= 0x3F & byte; 446ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 447ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 448ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootsize_t utf8_to_utf32_length(const char *src, size_t src_len) 449ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 450ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if (src == NULL || src_len == 0) { 451ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return 0; 452ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 453ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root size_t ret = 0; 454ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char* cur; 455ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char* end; 456ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root size_t num_to_skip; 457ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root for (cur = src, end = src + src_len, num_to_skip = 1; 458ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root cur < end; 459ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root cur += num_to_skip, ret++) { 460ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char first_char = *cur; 461ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root num_to_skip = 1; 462ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if ((first_char & 0x80) == 0) { // ASCII 463ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root continue; 464ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 465ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root int32_t mask; 466ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 467ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root for (mask = 0x40; (first_char & mask); num_to_skip++, mask >>= 1) { 468ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 469ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 470ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return ret; 471ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 472ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 473ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootvoid utf8_to_utf32(const char* src, size_t src_len, char32_t* dst) 474ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 475ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if (src == NULL || src_len == 0 || dst == NULL) { 476ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return; 477ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 478ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 479ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char* cur = src; 480ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const char* const end = src + src_len; 481ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root char32_t* cur_utf32 = dst; 482ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root while (cur < end) { 483ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root size_t num_read; 484ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root *cur_utf32++ = static_cast<char32_t>(utf32_at_internal(cur, &num_read)); 485ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root cur += num_read; 486ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 487ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root *cur_utf32 = 0; 488ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 489ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 490ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootstatic inline uint32_t utf8_to_utf32_codepoint(const uint8_t *src, size_t length) 491ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 492ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root uint32_t unicode; 493ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 494ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root switch (length) 495ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root { 496ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root case 1: 497ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return src[0]; 498ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root case 2: 499ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root unicode = src[0] & 0x1f; 500ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root utf8_shift_and_mask(&unicode, src[1]); 501ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return unicode; 502ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root case 3: 503ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root unicode = src[0] & 0x0f; 504ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root utf8_shift_and_mask(&unicode, src[1]); 505ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root utf8_shift_and_mask(&unicode, src[2]); 506ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return unicode; 507ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root case 4: 508ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root unicode = src[0] & 0x07; 509ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root utf8_shift_and_mask(&unicode, src[1]); 510ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root utf8_shift_and_mask(&unicode, src[2]); 511ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root utf8_shift_and_mask(&unicode, src[3]); 512ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return unicode; 513ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root default: 514ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return 0xffff; 515ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 516ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 517ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root //printf("Char at %p: len=%d, utf-16=%p\n", src, length, (void*)result); 518ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 519ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 520ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Rootssize_t utf8_to_utf16_length(const uint8_t* u8str, size_t u8len) 521ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 522ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const uint8_t* const u8end = u8str + u8len; 523ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const uint8_t* u8cur = u8str; 524ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 525ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root /* Validate that the UTF-8 is the correct len */ 526ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root size_t u16measuredLen = 0; 527ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root while (u8cur < u8end) { 528ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root u16measuredLen++; 529ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root int u8charLen = utf8_codepoint_len(*u8cur); 530ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root uint32_t codepoint = utf8_to_utf32_codepoint(u8cur, u8charLen); 531ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if (codepoint > 0xFFFF) u16measuredLen++; // this will be a surrogate pair in utf16 532ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root u8cur += u8charLen; 533ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 534ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 535ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root /** 536ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * Make sure that we ended where we thought we would and the output UTF-16 537ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root * will be exactly how long we were told it would be. 538ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root */ 539ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if (u8cur != u8end) { 540ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return -1; 541ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 542ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 543ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root return u16measuredLen; 544ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 545ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 546aa983c91caf848b9659599336a4156c28a125fa9Jeff Brownchar16_t* utf8_to_utf16_no_null_terminator(const uint8_t* u8str, size_t u8len, char16_t* u16str) 547ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root{ 548ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const uint8_t* const u8end = u8str + u8len; 549ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root const uint8_t* u8cur = u8str; 550ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root char16_t* u16cur = u16str; 551ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 552ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root while (u8cur < u8end) { 553ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root size_t u8len = utf8_codepoint_len(*u8cur); 554ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root uint32_t codepoint = utf8_to_utf32_codepoint(u8cur, u8len); 555ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 556ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root // Convert the UTF32 codepoint to one or more UTF16 codepoints 557ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root if (codepoint <= 0xFFFF) { 558ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root // Single UTF16 character 559ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root *u16cur++ = (char16_t) codepoint; 560ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } else { 561ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root // Multiple UTF16 characters with surrogates 562ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root codepoint = codepoint - 0x10000; 563ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root *u16cur++ = (char16_t) ((codepoint >> 10) + 0xD800); 564ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root *u16cur++ = (char16_t) ((codepoint & 0x3FF) + 0xDC00); 565ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 566ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 567ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root u8cur += u8len; 568ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root } 569aa983c91caf848b9659599336a4156c28a125fa9Jeff Brown return u16cur; 570aa983c91caf848b9659599336a4156c28a125fa9Jeff Brown} 571aa983c91caf848b9659599336a4156c28a125fa9Jeff Brown 572aa983c91caf848b9659599336a4156c28a125fa9Jeff Brownvoid utf8_to_utf16(const uint8_t* u8str, size_t u8len, char16_t* u16str) { 573aa983c91caf848b9659599336a4156c28a125fa9Jeff Brown char16_t* end = utf8_to_utf16_no_null_terminator(u8str, u8len, u16str); 574aa983c91caf848b9659599336a4156c28a125fa9Jeff Brown *end = 0; 575ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 576ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root 5770f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackbornchar16_t* utf8_to_utf16_n(const uint8_t* src, size_t srcLen, char16_t* dst, size_t dstLen) { 5780f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn const uint8_t* const u8end = src + srcLen; 5790f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn const uint8_t* u8cur = src; 5805bed8036644f552210a7cfcbed2d6d20cf2981b0Mark Salyzyn const char16_t* const u16end = dst + dstLen; 5810f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn char16_t* u16cur = dst; 5820f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn 5830f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn while (u8cur < u8end && u16cur < u16end) { 5840f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn size_t u8len = utf8_codepoint_len(*u8cur); 5850f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn uint32_t codepoint = utf8_to_utf32_codepoint(u8cur, u8len); 5860f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn 5870f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn // Convert the UTF32 codepoint to one or more UTF16 codepoints 5880f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn if (codepoint <= 0xFFFF) { 5890f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn // Single UTF16 character 5900f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn *u16cur++ = (char16_t) codepoint; 5910f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn } else { 5920f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn // Multiple UTF16 characters with surrogates 5930f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn codepoint = codepoint - 0x10000; 5940f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn *u16cur++ = (char16_t) ((codepoint >> 10) + 0xD800); 5950f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn if (u16cur >= u16end) { 5960f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn // Ooops... not enough room for this surrogate pair. 5970f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn return u16cur-1; 5980f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn } 5990f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn *u16cur++ = (char16_t) ((codepoint & 0x3FF) + 0xDC00); 6000f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn } 6010f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn 6020f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn u8cur += u8len; 6030f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn } 6040f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn return u16cur; 6050f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn} 6060f10d0abf3e6f6b5631c091256f8b4e7a20a33d0Dianne Hackborn 607ba0165bef09729a33ab8e0ca329342be05e0d859Kenny Root} 608