1c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// Copyright 2007, Google Inc. 2c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// All rights reserved. 3c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// 4c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// Redistribution and use in source and binary forms, with or without 5c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// modification, are permitted provided that the following conditions are 6c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// met: 7c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// 8c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// * Redistributions of source code must retain the above copyright 9c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// notice, this list of conditions and the following disclaimer. 10c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// * Redistributions in binary form must reproduce the above 11c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// copyright notice, this list of conditions and the following disclaimer 12c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// in the documentation and/or other materials provided with the 13c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// distribution. 14c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// * Neither the name of Google Inc. nor the names of its 15c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// contributors may be used to endorse or promote products derived from 16c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// this software without specific prior written permission. 17c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// 18c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 30c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#include "googleurl/src/url_canon.h" 31c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott#include "googleurl/src/url_canon_internal.h" 32c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 33c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// Query canonicalization in IE 34c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// ---------------------------- 35c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// IE is very permissive for query parameters specified in links on the page 36c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// (in contrast to links that it constructs itself based on form data). It does 37c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// not unescape any character. It does not reject any escape sequence (be they 38c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// invalid like "%2y" or freaky like %00). 39c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// 40c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// IE only escapes spaces and nothing else. Embedded NULLs, tabs (0x09), 41c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// LF (0x0a), and CR (0x0d) are removed (this probably happens at an earlier 42c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// layer since they are removed from all portions of the URL). All other 43c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// characters are passed unmodified. Invalid UTF-16 sequences are preserved as 44c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// well, with each character in the input being converted to UTF-8. It is the 45c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// server's job to make sense of this invalid query. 46c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// 47c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// Invalid multibyte sequences (for example, invalid UTF-8 on a UTF-8 page) 48c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// are converted to the invalid character and sent as unescaped UTF-8 (0xef, 49c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// 0xbf, 0xbd). This may not be canonicalization, the parser may generate these 50c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// strings before the URL handler ever sees them. 51c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// 52c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// Our query canonicalization 53c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// -------------------------- 54c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// We escape all non-ASCII characters and control characters, like Firefox. 55c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// This is more conformant to the URL spec, and there do not seem to be many 56c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// problems relating to Firefox's behavior. 57c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// 58c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// Like IE, we will never unescape (although the application may want to try 59c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// unescaping to present the user with a more understandable URL). We will 60c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// replace all invalid sequences (including invalid UTF-16 sequences, which IE 61c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// doesn't) with the "invalid character," and we will escape it. 62c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 63c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottnamespace url_canon { 64c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 65c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottnamespace { 66c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 67c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// Returns true if the characters starting at |begin| and going until |end| 68c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// (non-inclusive) are all representable in 7-bits. 69c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scotttemplate<typename CHAR, typename UCHAR> 70c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottbool IsAllASCII(const CHAR* spec, const url_parse::Component& query) { 71c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott int end = query.end(); 72c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott for (int i = query.begin; i < end; i++) { 73c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott if (static_cast<UCHAR>(spec[i]) >= 0x80) 74c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return false; 75c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott } 76c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return true; 77c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 78c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 79c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// Appends the given string to the output, escaping characters that do not 80c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// match the given |type| in SharedCharTypes. This version will accept 8 or 16 81c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// bit characters, but assumes that they have only 7-bit values. It also assumes 82c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// that all UTF-8 values are correct, so doesn't bother checking 83c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scotttemplate<typename CHAR> 84c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottvoid AppendRaw8BitQueryString(const CHAR* source, int length, 85c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott CanonOutput* output) { 86c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott for (int i = 0; i < length; i++) { 87c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott if (!IsQueryChar(static_cast<unsigned char>(source[i]))) 88c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott AppendEscapedChar(static_cast<unsigned char>(source[i]), output); 89c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott else // Doesn't need escaping. 90c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott output->push_back(static_cast<char>(source[i])); 91c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott } 92c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 93c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 94c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// Runs the converter on the given UTF-8 input. Since the converter expects 95c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// UTF-16, we have to convert first. The converter must be non-NULL. 96c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottvoid RunConverter(const char* spec, 97c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott const url_parse::Component& query, 98c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott CharsetConverter* converter, 99c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott CanonOutput* output) { 100c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // This function will replace any misencoded values with the invalid 101c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // character. This is what we want so we don't have to check for error. 102c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott RawCanonOutputW<1024> utf16; 103c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott ConvertUTF8ToUTF16(&spec[query.begin], query.len, &utf16); 104c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott converter->ConvertFromUTF16(utf16.data(), utf16.length(), output); 105c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 106c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 107c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// Runs the converter with the given UTF-16 input. We don't have to do 108c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// anything, but this overriddden function allows us to use the same code 109c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott// for both UTF-8 and UTF-16 input. 110c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottvoid RunConverter(const char16* spec, 111c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott const url_parse::Component& query, 112c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott CharsetConverter* converter, 113c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott CanonOutput* output) { 114c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott converter->ConvertFromUTF16(&spec[query.begin], query.len, output); 115c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 116c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 117c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scotttemplate<typename CHAR, typename UCHAR> 118c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottvoid DoConvertToQueryEncoding(const CHAR* spec, 119c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott const url_parse::Component& query, 120c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott CharsetConverter* converter, 121c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott CanonOutput* output) { 122c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott if (IsAllASCII<CHAR, UCHAR>(spec, query)) { 123c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // Easy: the input can just appended with no character set conversions. 124c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott AppendRaw8BitQueryString(&spec[query.begin], query.len, output); 125c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 126c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott } else { 127c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // Harder: convert to the proper encoding first. 128c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott if (converter) { 129c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // Run the converter to get an 8-bit string, then append it, escaping 130c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // necessary values. 131c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott RawCanonOutput<1024> eight_bit; 132c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott RunConverter(spec, query, converter, &eight_bit); 133c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott AppendRaw8BitQueryString(eight_bit.data(), eight_bit.length(), output); 134c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 135c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott } else { 136c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott // No converter, do our own UTF-8 conversion. 137c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott AppendStringOfType(&spec[query.begin], query.len, CHAR_QUERY, output); 138c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott } 139c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott } 140c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 141c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 142c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scotttemplate<typename CHAR, typename UCHAR> 143c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottvoid DoCanonicalizeQuery(const CHAR* spec, 144c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott const url_parse::Component& query, 145c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott CharsetConverter* converter, 146c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott CanonOutput* output, 147c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott url_parse::Component* out_query) { 148c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott if (query.len < 0) { 149c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott *out_query = url_parse::Component(); 150c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott return; 151c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott } 152c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 153c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott output->push_back('?'); 154c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott out_query->begin = output->length(); 155c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 156c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott DoConvertToQueryEncoding<CHAR, UCHAR>(spec, query, converter, output); 157c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 158c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott out_query->len = output->length() - out_query->begin; 159c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 160c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 161c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} // namespace 162c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 163c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottvoid CanonicalizeQuery(const char* spec, 164c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott const url_parse::Component& query, 165c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott CharsetConverter* converter, 166c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott CanonOutput* output, 167c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott url_parse::Component* out_query) { 168c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott DoCanonicalizeQuery<char, unsigned char>(spec, query, converter, 169c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott output, out_query); 170c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 171c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 172c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottvoid CanonicalizeQuery(const char16* spec, 173c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott const url_parse::Component& query, 174c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott CharsetConverter* converter, 175c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott CanonOutput* output, 176c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott url_parse::Component* out_query) { 177c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott DoCanonicalizeQuery<char16, char16>(spec, query, converter, 178c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott output, out_query); 179c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 180c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 181c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scottvoid ConvertUTF16ToQueryEncoding(const char16* input, 182c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott const url_parse::Component& query, 183c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott CharsetConverter* converter, 184c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott CanonOutput* output) { 185c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott DoConvertToQueryEncoding<char16, char16>(input, query, 186c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott converter, output); 187c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} 188c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott 189c7f5f8508d98d5952d42ed7648c2a8f30a4da156Patrick Scott} // namespace url_canon 190