StringPool.cpp revision 46fc395386f2277a0425b2c46be08fd91f6554f4
1// 2// Copyright 2006 The Android Open Source Project 3// 4// Build resource files from raw assets. 5// 6 7#include "StringPool.h" 8#include "ResourceTable.h" 9 10#include <utils/ByteOrder.h> 11#include <utils/SortedVector.h> 12 13#if HAVE_PRINTF_ZD 14# define ZD "%zd" 15# define ZD_TYPE ssize_t 16#else 17# define ZD "%ld" 18# define ZD_TYPE long 19#endif 20 21#define NOISY(x) //x 22 23void strcpy16_htod(uint16_t* dst, const uint16_t* src) 24{ 25 while (*src) { 26 char16_t s = htods(*src); 27 *dst++ = s; 28 src++; 29 } 30 *dst = 0; 31} 32 33void printStringPool(const ResStringPool* pool) 34{ 35 SortedVector<const void*> uniqueStrings; 36 const size_t N = pool->size(); 37 for (size_t i=0; i<N; i++) { 38 size_t len; 39 if (pool->isUTF8()) { 40 uniqueStrings.add(pool->string8At(i, &len)); 41 } else { 42 uniqueStrings.add(pool->stringAt(i, &len)); 43 } 44 } 45 46 printf("String pool of " ZD " unique %s %s strings, " ZD " entries and " 47 ZD " styles using " ZD " bytes:\n", 48 (ZD_TYPE)uniqueStrings.size(), pool->isUTF8() ? "UTF-8" : "UTF-16", 49 pool->isSorted() ? "sorted" : "non-sorted", 50 (ZD_TYPE)N, (ZD_TYPE)pool->styleCount(), (ZD_TYPE)pool->bytes()); 51 52 const size_t NS = pool->size(); 53 for (size_t s=0; s<NS; s++) { 54 String8 str = pool->string8ObjectAt(s); 55 printf("String #" ZD ": %s\n", (ZD_TYPE) s, str.string()); 56 } 57} 58 59String8 StringPool::entry::makeConfigsString() const { 60 String8 configStr(configTypeName); 61 if (configStr.size() > 0) configStr.append(" "); 62 if (configs.size() > 0) { 63 for (size_t j=0; j<configs.size(); j++) { 64 if (j > 0) configStr.append(", "); 65 configStr.append(configs[j].toString()); 66 } 67 } else { 68 configStr = "(none)"; 69 } 70 return configStr; 71} 72 73int StringPool::entry::compare(const entry& o) const { 74 // Strings with styles go first, to reduce the size of the styles array. 75 // We don't care about the relative order of these strings. 76 if (hasStyles) { 77 return o.hasStyles ? 0 : -1; 78 } 79 if (o.hasStyles) { 80 return 1; 81 } 82 83 // Sort unstyled strings by type, then by logical configuration. 84 int comp = configTypeName.compare(o.configTypeName); 85 if (comp != 0) { 86 return comp; 87 } 88 const size_t LHN = configs.size(); 89 const size_t RHN = o.configs.size(); 90 size_t i=0; 91 while (i < LHN && i < RHN) { 92 comp = configs[i].compareLogical(o.configs[i]); 93 if (comp != 0) { 94 return comp; 95 } 96 i++; 97 } 98 if (LHN < RHN) return -1; 99 else if (LHN > RHN) return 1; 100 return 0; 101} 102 103StringPool::StringPool(bool utf8) : 104 mUTF8(utf8), mValues(-1) 105{ 106} 107 108ssize_t StringPool::add(const String16& value, const Vector<entry_style_span>& spans, 109 const String8* configTypeName, const ResTable_config* config) 110{ 111 ssize_t res = add(value, false, configTypeName, config); 112 if (res >= 0) { 113 addStyleSpans(res, spans); 114 } 115 return res; 116} 117 118ssize_t StringPool::add(const String16& value, 119 bool mergeDuplicates, const String8* configTypeName, const ResTable_config* config) 120{ 121 ssize_t vidx = mValues.indexOfKey(value); 122 ssize_t pos = vidx >= 0 ? mValues.valueAt(vidx) : -1; 123 ssize_t eidx = pos >= 0 ? mEntryArray.itemAt(pos) : -1; 124 if (eidx < 0) { 125 eidx = mEntries.add(entry(value)); 126 if (eidx < 0) { 127 fprintf(stderr, "Failure adding string %s\n", String8(value).string()); 128 return eidx; 129 } 130 } 131 132 if (configTypeName != NULL) { 133 entry& ent = mEntries.editItemAt(eidx); 134 NOISY(printf("*** adding config type name %s, was %s\n", 135 configTypeName->string(), ent.configTypeName.string())); 136 if (ent.configTypeName.size() <= 0) { 137 ent.configTypeName = *configTypeName; 138 } else if (ent.configTypeName != *configTypeName) { 139 ent.configTypeName = " "; 140 } 141 } 142 143 if (config != NULL) { 144 // Add this to the set of configs associated with the string. 145 entry& ent = mEntries.editItemAt(eidx); 146 size_t addPos; 147 for (addPos=0; addPos<ent.configs.size(); addPos++) { 148 int cmp = ent.configs.itemAt(addPos).compareLogical(*config); 149 if (cmp >= 0) { 150 if (cmp > 0) { 151 NOISY(printf("*** inserting config: %s\n", config->toString().string())); 152 ent.configs.insertAt(*config, addPos); 153 } 154 break; 155 } 156 } 157 if (addPos >= ent.configs.size()) { 158 NOISY(printf("*** adding config: %s\n", config->toString().string())); 159 ent.configs.add(*config); 160 } 161 } 162 163 const bool first = vidx < 0; 164 const bool styled = (pos >= 0 && (size_t)pos < mEntryStyleArray.size()) ? 165 mEntryStyleArray[pos].spans.size() : 0; 166 if (first || styled || !mergeDuplicates) { 167 pos = mEntryArray.add(eidx); 168 if (first) { 169 vidx = mValues.add(value, pos); 170 } 171 entry& ent = mEntries.editItemAt(eidx); 172 ent.indices.add(pos); 173 } 174 175 NOISY(printf("Adding string %s to pool: pos=%d eidx=%d vidx=%d\n", 176 String8(value).string(), pos, eidx, vidx)); 177 178 return pos; 179} 180 181status_t StringPool::addStyleSpan(size_t idx, const String16& name, 182 uint32_t start, uint32_t end) 183{ 184 entry_style_span span; 185 span.name = name; 186 span.span.firstChar = start; 187 span.span.lastChar = end; 188 return addStyleSpan(idx, span); 189} 190 191status_t StringPool::addStyleSpans(size_t idx, const Vector<entry_style_span>& spans) 192{ 193 const size_t N=spans.size(); 194 for (size_t i=0; i<N; i++) { 195 status_t err = addStyleSpan(idx, spans[i]); 196 if (err != NO_ERROR) { 197 return err; 198 } 199 } 200 return NO_ERROR; 201} 202 203status_t StringPool::addStyleSpan(size_t idx, const entry_style_span& span) 204{ 205 // Place blank entries in the span array up to this index. 206 while (mEntryStyleArray.size() <= idx) { 207 mEntryStyleArray.add(); 208 } 209 210 entry_style& style = mEntryStyleArray.editItemAt(idx); 211 style.spans.add(span); 212 mEntries.editItemAt(mEntryArray[idx]).hasStyles = true; 213 return NO_ERROR; 214} 215 216#ifdef __GLIBC__ 217int StringPool::config_sort(const void* lhs, const void* rhs, void* state) 218#else 219int StringPool::config_sort(void* state, const void* lhs, const void* rhs) 220#endif 221{ 222 StringPool* pool = (StringPool*)state; 223 const entry& lhe = pool->mEntries[pool->mEntryArray[*static_cast<const size_t*>(lhs)]]; 224 const entry& rhe = pool->mEntries[pool->mEntryArray[*static_cast<const size_t*>(rhs)]]; 225 return lhe.compare(rhe); 226} 227 228void StringPool::sortByConfig() 229{ 230 LOG_ALWAYS_FATAL_IF(mOriginalPosToNewPos.size() > 0, "Can't sort string pool after already sorted."); 231 232 const size_t N = mEntryArray.size(); 233 234 // This is a vector that starts out with a 1:1 mapping to entries 235 // in the array, which we will sort to come up with the desired order. 236 // At that point it maps from the new position in the array to the 237 // original position the entry appeared. 238 Vector<size_t> newPosToOriginalPos; 239 newPosToOriginalPos.setCapacity(N); 240 for (size_t i=0; i < N; i++) { 241 newPosToOriginalPos.add(i); 242 } 243 244 // Sort the array. 245 NOISY(printf("SORTING STRINGS BY CONFIGURATION...\n")); 246 // Vector::sort uses insertion sort, which is very slow for this data set. 247 // Use quicksort instead because we don't need a stable sort here. 248 // For more fun, GLibC took qsort_r from BSD but then decided to swap the 249 // order the last two parameters. 250#ifdef __GLIBC__ 251 qsort_r(newPosToOriginalPos.editArray(), N, sizeof(size_t), config_sort, this); 252#else 253 qsort_r(newPosToOriginalPos.editArray(), N, sizeof(size_t), this, config_sort); 254#endif 255 //newPosToOriginalPos.sort(config_sort, this); 256 NOISY(printf("DONE SORTING STRINGS BY CONFIGURATION.\n")); 257 258 // Create the reverse mapping from the original position in the array 259 // to the new position where it appears in the sorted array. This is 260 // so that clients can re-map any positions they had previously stored. 261 mOriginalPosToNewPos = newPosToOriginalPos; 262 for (size_t i=0; i<N; i++) { 263 mOriginalPosToNewPos.editItemAt(newPosToOriginalPos[i]) = i; 264 } 265 266#if 0 267 SortedVector<entry> entries; 268 269 for (size_t i=0; i<N; i++) { 270 printf("#%d was %d: %s\n", i, newPosToOriginalPos[i], 271 mEntries[mEntryArray[newPosToOriginalPos[i]]].makeConfigsString().string()); 272 entries.add(mEntries[mEntryArray[i]]); 273 } 274 275 for (size_t i=0; i<entries.size(); i++) { 276 printf("Sorted config #%d: %s\n", i, 277 entries[i].makeConfigsString().string()); 278 } 279#endif 280 281 // Now we rebuild the arrays. 282 Vector<entry> newEntries; 283 Vector<size_t> newEntryArray; 284 Vector<entry_style> newEntryStyleArray; 285 DefaultKeyedVector<size_t, size_t> origOffsetToNewOffset; 286 287 for (size_t i=0; i<N; i++) { 288 // We are filling in new offset 'i'; oldI is where we can find it 289 // in the original data structure. 290 size_t oldI = newPosToOriginalPos[i]; 291 // This is the actual entry associated with the old offset. 292 const entry& oldEnt = mEntries[mEntryArray[oldI]]; 293 // This is the same entry the last time we added it to the 294 // new entry array, if any. 295 ssize_t newIndexOfOffset = origOffsetToNewOffset.indexOfKey(oldI); 296 size_t newOffset; 297 if (newIndexOfOffset < 0) { 298 // This is the first time we have seen the entry, so add 299 // it. 300 newOffset = newEntries.add(oldEnt); 301 newEntries.editItemAt(newOffset).indices.clear(); 302 } else { 303 // We have seen this entry before, use the existing one 304 // instead of adding it again. 305 newOffset = origOffsetToNewOffset.valueAt(newIndexOfOffset); 306 } 307 // Update the indices to include this new position. 308 newEntries.editItemAt(newOffset).indices.add(i); 309 // And add the offset of the entry to the new entry array. 310 newEntryArray.add(newOffset); 311 // Add any old style to the new style array. 312 if (mEntryStyleArray.size() > 0) { 313 if (oldI < mEntryStyleArray.size()) { 314 newEntryStyleArray.add(mEntryStyleArray[oldI]); 315 } else { 316 newEntryStyleArray.add(entry_style()); 317 } 318 } 319 } 320 321 // Now trim any entries at the end of the new style array that are 322 // not needed. 323 for (ssize_t i=newEntryStyleArray.size()-1; i>=0; i--) { 324 const entry_style& style = newEntryStyleArray[i]; 325 if (style.spans.size() > 0) { 326 // That's it. 327 break; 328 } 329 // This one is not needed; remove. 330 newEntryStyleArray.removeAt(i); 331 } 332 333 // All done, install the new data structures and upate mValues with 334 // the new positions. 335 mEntries = newEntries; 336 mEntryArray = newEntryArray; 337 mEntryStyleArray = newEntryStyleArray; 338 mValues.clear(); 339 for (size_t i=0; i<mEntries.size(); i++) { 340 const entry& ent = mEntries[i]; 341 mValues.add(ent.value, ent.indices[0]); 342 } 343 344#if 0 345 printf("FINAL SORTED STRING CONFIGS:\n"); 346 for (size_t i=0; i<mEntries.size(); i++) { 347 const entry& ent = mEntries[i]; 348 printf("#" ZD " %s: %s\n", (ZD_TYPE)i, ent.makeConfigsString().string(), 349 String8(ent.value).string()); 350 } 351#endif 352} 353 354sp<AaptFile> StringPool::createStringBlock() 355{ 356 sp<AaptFile> pool = new AaptFile(String8(), AaptGroupEntry(), 357 String8()); 358 status_t err = writeStringBlock(pool); 359 return err == NO_ERROR ? pool : NULL; 360} 361 362#define ENCODE_LENGTH(str, chrsz, strSize) \ 363{ \ 364 size_t maxMask = 1 << ((chrsz*8)-1); \ 365 size_t maxSize = maxMask-1; \ 366 if (strSize > maxSize) { \ 367 *str++ = maxMask | ((strSize>>(chrsz*8))&maxSize); \ 368 } \ 369 *str++ = strSize; \ 370} 371 372status_t StringPool::writeStringBlock(const sp<AaptFile>& pool) 373{ 374 // Allow appending. Sorry this is a little wacky. 375 if (pool->getSize() > 0) { 376 sp<AaptFile> block = createStringBlock(); 377 if (block == NULL) { 378 return UNKNOWN_ERROR; 379 } 380 ssize_t res = pool->writeData(block->getData(), block->getSize()); 381 return (res >= 0) ? (status_t)NO_ERROR : res; 382 } 383 384 // First we need to add all style span names to the string pool. 385 // We do this now (instead of when the span is added) so that these 386 // will appear at the end of the pool, not disrupting the order 387 // our client placed their own strings in it. 388 389 const size_t STYLES = mEntryStyleArray.size(); 390 size_t i; 391 392 for (i=0; i<STYLES; i++) { 393 entry_style& style = mEntryStyleArray.editItemAt(i); 394 const size_t N = style.spans.size(); 395 for (size_t i=0; i<N; i++) { 396 entry_style_span& span = style.spans.editItemAt(i); 397 ssize_t idx = add(span.name, true); 398 if (idx < 0) { 399 fprintf(stderr, "Error adding span for style tag '%s'\n", 400 String8(span.name).string()); 401 return idx; 402 } 403 span.span.name.index = (uint32_t)idx; 404 } 405 } 406 407 const size_t ENTRIES = mEntryArray.size(); 408 409 // Now build the pool of unique strings. 410 411 const size_t STRINGS = mEntries.size(); 412 const size_t preSize = sizeof(ResStringPool_header) 413 + (sizeof(uint32_t)*ENTRIES) 414 + (sizeof(uint32_t)*STYLES); 415 if (pool->editData(preSize) == NULL) { 416 fprintf(stderr, "ERROR: Out of memory for string pool\n"); 417 return NO_MEMORY; 418 } 419 420 const size_t charSize = mUTF8 ? sizeof(uint8_t) : sizeof(char16_t); 421 422 size_t strPos = 0; 423 for (i=0; i<STRINGS; i++) { 424 entry& ent = mEntries.editItemAt(i); 425 const size_t strSize = (ent.value.size()); 426 const size_t lenSize = strSize > (size_t)(1<<((charSize*8)-1))-1 ? 427 charSize*2 : charSize; 428 429 String8 encStr; 430 if (mUTF8) { 431 encStr = String8(ent.value); 432 } 433 434 const size_t encSize = mUTF8 ? encStr.size() : 0; 435 const size_t encLenSize = mUTF8 ? 436 (encSize > (size_t)(1<<((charSize*8)-1))-1 ? 437 charSize*2 : charSize) : 0; 438 439 ent.offset = strPos; 440 441 const size_t totalSize = lenSize + encLenSize + 442 ((mUTF8 ? encSize : strSize)+1)*charSize; 443 444 void* dat = (void*)pool->editData(preSize + strPos + totalSize); 445 if (dat == NULL) { 446 fprintf(stderr, "ERROR: Out of memory for string pool\n"); 447 return NO_MEMORY; 448 } 449 dat = (uint8_t*)dat + preSize + strPos; 450 if (mUTF8) { 451 uint8_t* strings = (uint8_t*)dat; 452 453 ENCODE_LENGTH(strings, sizeof(uint8_t), strSize) 454 455 ENCODE_LENGTH(strings, sizeof(uint8_t), encSize) 456 457 strncpy((char*)strings, encStr, encSize+1); 458 } else { 459 uint16_t* strings = (uint16_t*)dat; 460 461 ENCODE_LENGTH(strings, sizeof(uint16_t), strSize) 462 463 strcpy16_htod(strings, ent.value); 464 } 465 466 strPos += totalSize; 467 } 468 469 // Pad ending string position up to a uint32_t boundary. 470 471 if (strPos&0x3) { 472 size_t padPos = ((strPos+3)&~0x3); 473 uint8_t* dat = (uint8_t*)pool->editData(preSize + padPos); 474 if (dat == NULL) { 475 fprintf(stderr, "ERROR: Out of memory padding string pool\n"); 476 return NO_MEMORY; 477 } 478 memset(dat+preSize+strPos, 0, padPos-strPos); 479 strPos = padPos; 480 } 481 482 // Build the pool of style spans. 483 484 size_t styPos = strPos; 485 for (i=0; i<STYLES; i++) { 486 entry_style& ent = mEntryStyleArray.editItemAt(i); 487 const size_t N = ent.spans.size(); 488 const size_t totalSize = (N*sizeof(ResStringPool_span)) 489 + sizeof(ResStringPool_ref); 490 491 ent.offset = styPos-strPos; 492 uint8_t* dat = (uint8_t*)pool->editData(preSize + styPos + totalSize); 493 if (dat == NULL) { 494 fprintf(stderr, "ERROR: Out of memory for string styles\n"); 495 return NO_MEMORY; 496 } 497 ResStringPool_span* span = (ResStringPool_span*)(dat+preSize+styPos); 498 for (size_t i=0; i<N; i++) { 499 span->name.index = htodl(ent.spans[i].span.name.index); 500 span->firstChar = htodl(ent.spans[i].span.firstChar); 501 span->lastChar = htodl(ent.spans[i].span.lastChar); 502 span++; 503 } 504 span->name.index = htodl(ResStringPool_span::END); 505 506 styPos += totalSize; 507 } 508 509 if (STYLES > 0) { 510 // Add full terminator at the end (when reading we validate that 511 // the end of the pool is fully terminated to simplify error 512 // checking). 513 size_t extra = sizeof(ResStringPool_span)-sizeof(ResStringPool_ref); 514 uint8_t* dat = (uint8_t*)pool->editData(preSize + styPos + extra); 515 if (dat == NULL) { 516 fprintf(stderr, "ERROR: Out of memory for string styles\n"); 517 return NO_MEMORY; 518 } 519 uint32_t* p = (uint32_t*)(dat+preSize+styPos); 520 while (extra > 0) { 521 *p++ = htodl(ResStringPool_span::END); 522 extra -= sizeof(uint32_t); 523 } 524 styPos += extra; 525 } 526 527 // Write header. 528 529 ResStringPool_header* header = 530 (ResStringPool_header*)pool->padData(sizeof(uint32_t)); 531 if (header == NULL) { 532 fprintf(stderr, "ERROR: Out of memory for string pool\n"); 533 return NO_MEMORY; 534 } 535 memset(header, 0, sizeof(*header)); 536 header->header.type = htods(RES_STRING_POOL_TYPE); 537 header->header.headerSize = htods(sizeof(*header)); 538 header->header.size = htodl(pool->getSize()); 539 header->stringCount = htodl(ENTRIES); 540 header->styleCount = htodl(STYLES); 541 if (mUTF8) { 542 header->flags |= htodl(ResStringPool_header::UTF8_FLAG); 543 } 544 header->stringsStart = htodl(preSize); 545 header->stylesStart = htodl(STYLES > 0 ? (preSize+strPos) : 0); 546 547 // Write string index array. 548 549 uint32_t* index = (uint32_t*)(header+1); 550 for (i=0; i<ENTRIES; i++) { 551 entry& ent = mEntries.editItemAt(mEntryArray[i]); 552 *index++ = htodl(ent.offset); 553 NOISY(printf("Writing entry #%d: \"%s\" ent=%d off=%d\n", i, 554 String8(ent.value).string(), 555 mEntryArray[i], ent.offset)); 556 } 557 558 // Write style index array. 559 560 for (i=0; i<STYLES; i++) { 561 *index++ = htodl(mEntryStyleArray[i].offset); 562 } 563 564 return NO_ERROR; 565} 566 567ssize_t StringPool::offsetForString(const String16& val) const 568{ 569 const Vector<size_t>* indices = offsetsForString(val); 570 ssize_t res = indices != NULL && indices->size() > 0 ? indices->itemAt(0) : -1; 571 NOISY(printf("Offset for string %s: %d (%s)\n", String8(val).string(), res, 572 res >= 0 ? String8(mEntries[mEntryArray[res]].value).string() : String8())); 573 return res; 574} 575 576const Vector<size_t>* StringPool::offsetsForString(const String16& val) const 577{ 578 ssize_t pos = mValues.valueFor(val); 579 if (pos < 0) { 580 return NULL; 581 } 582 return &mEntries[mEntryArray[pos]].indices; 583} 584