1/* 2 ********************************************************************** 3 * Copyright (C) 2005-2012, International Business Machines 4 * Corporation and others. All Rights Reserved. 5 ********************************************************************** 6 */ 7 8#include "unicode/utypes.h" 9 10#if !UCONFIG_NO_CONVERSION 11 12#include "csrucode.h" 13#include "csmatch.h" 14 15U_NAMESPACE_BEGIN 16 17CharsetRecog_Unicode::~CharsetRecog_Unicode() 18{ 19 // nothing to do 20} 21 22CharsetRecog_UTF_16_BE::~CharsetRecog_UTF_16_BE() 23{ 24 // nothing to do 25} 26 27const char *CharsetRecog_UTF_16_BE::getName() const 28{ 29 return "UTF-16BE"; 30} 31 32UBool CharsetRecog_UTF_16_BE::match(InputText* textIn, CharsetMatch *results) const 33{ 34 const uint8_t *input = textIn->fRawInput; 35 int32_t confidence = 0; 36 37 if (input[0] == 0xFE && input[1] == 0xFF) { 38 confidence = 100; 39 } 40 41 // TODO: Do some statastics to check for unsigned UTF-16BE 42 results->set(textIn, this, confidence); 43 return (confidence > 0); 44} 45 46CharsetRecog_UTF_16_LE::~CharsetRecog_UTF_16_LE() 47{ 48 // nothing to do 49} 50 51const char *CharsetRecog_UTF_16_LE::getName() const 52{ 53 return "UTF-16LE"; 54} 55 56UBool CharsetRecog_UTF_16_LE::match(InputText* textIn, CharsetMatch *results) const 57{ 58 const uint8_t *input = textIn->fRawInput; 59 int32_t confidence = 0; 60 61 if (input[0] == 0xFF && input[1] == 0xFE && (input[2] != 0x00 || input[3] != 0x00)) { 62 confidence = 100; 63 } 64 65 // TODO: Do some statastics to check for unsigned UTF-16LE 66 results->set(textIn, this, confidence); 67 return (confidence > 0); 68} 69 70CharsetRecog_UTF_32::~CharsetRecog_UTF_32() 71{ 72 // nothing to do 73} 74 75UBool CharsetRecog_UTF_32::match(InputText* textIn, CharsetMatch *results) const 76{ 77 const uint8_t *input = textIn->fRawInput; 78 int32_t limit = (textIn->fRawLength / 4) * 4; 79 int32_t numValid = 0; 80 int32_t numInvalid = 0; 81 bool hasBOM = FALSE; 82 int32_t confidence = 0; 83 84 if (getChar(input, 0) == 0x0000FEFFUL) { 85 hasBOM = TRUE; 86 } 87 88 for(int32_t i = 0; i < limit; i += 4) { 89 int32_t ch = getChar(input, i); 90 91 if (ch < 0 || ch >= 0x10FFFF || (ch >= 0xD800 && ch <= 0xDFFF)) { 92 numInvalid += 1; 93 } else { 94 numValid += 1; 95 } 96 } 97 98 99 // Cook up some sort of confidence score, based on presense of a BOM 100 // and the existence of valid and/or invalid multi-byte sequences. 101 if (hasBOM && numInvalid==0) { 102 confidence = 100; 103 } else if (hasBOM && numValid > numInvalid*10) { 104 confidence = 80; 105 } else if (numValid > 3 && numInvalid == 0) { 106 confidence = 100; 107 } else if (numValid > 0 && numInvalid == 0) { 108 confidence = 80; 109 } else if (numValid > numInvalid*10) { 110 // Probably corruput UTF-32BE data. Valid sequences aren't likely by chance. 111 confidence = 25; 112 } 113 114 results->set(textIn, this, confidence); 115 return (confidence > 0); 116} 117 118CharsetRecog_UTF_32_BE::~CharsetRecog_UTF_32_BE() 119{ 120 // nothing to do 121} 122 123const char *CharsetRecog_UTF_32_BE::getName() const 124{ 125 return "UTF-32BE"; 126} 127 128int32_t CharsetRecog_UTF_32_BE::getChar(const uint8_t *input, int32_t index) const 129{ 130 return input[index + 0] << 24 | input[index + 1] << 16 | 131 input[index + 2] << 8 | input[index + 3]; 132} 133 134CharsetRecog_UTF_32_LE::~CharsetRecog_UTF_32_LE() 135{ 136 // nothing to do 137} 138 139const char *CharsetRecog_UTF_32_LE::getName() const 140{ 141 return "UTF-32LE"; 142} 143 144int32_t CharsetRecog_UTF_32_LE::getChar(const uint8_t *input, int32_t index) const 145{ 146 return input[index + 3] << 24 | input[index + 2] << 16 | 147 input[index + 1] << 8 | input[index + 0]; 148} 149 150U_NAMESPACE_END 151#endif 152 153