string_tokenizer_unittest.cc revision c7f5f8508d98d5952d42ed7648c2a8f30a4da156 
1// Copyright (c) 2006-2008 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/string_tokenizer.h"
6#include "testing/gtest/include/gtest/gtest.h"
7
8using std::string;
9
10namespace {
11class StringTokenizerTest : public testing::Test {};
12}
13
14TEST(StringTokenizerTest, Simple) {
15  string input = "this is a test";
16  StringTokenizer t(input, " ");
17
18  EXPECT_TRUE(t.GetNext());
19  EXPECT_EQ(string("this"), t.token());
20
21  EXPECT_TRUE(t.GetNext());
22  EXPECT_EQ(string("is"), t.token());
23
24  EXPECT_TRUE(t.GetNext());
25  EXPECT_EQ(string("a"), t.token());
26
27  EXPECT_TRUE(t.GetNext());
28  EXPECT_EQ(string("test"), t.token());
29
30  EXPECT_FALSE(t.GetNext());
31}
32
33TEST(StringTokenizerTest, Reset) {
34  string input = "this is a test";
35  StringTokenizer t(input, " ");
36
37  for (int i = 0; i < 2; ++i) {
38    EXPECT_TRUE(t.GetNext());
39    EXPECT_EQ(string("this"), t.token());
40
41    EXPECT_TRUE(t.GetNext());
42    EXPECT_EQ(string("is"), t.token());
43
44    EXPECT_TRUE(t.GetNext());
45    EXPECT_EQ(string("a"), t.token());
46
47    EXPECT_TRUE(t.GetNext());
48    EXPECT_EQ(string("test"), t.token());
49
50    EXPECT_FALSE(t.GetNext());
51    t.Reset();
52  }
53}
54
55TEST(StringTokenizerTest, RetDelims) {
56  string input = "this is a test";
57  StringTokenizer t(input, " ");
58  t.set_options(StringTokenizer::RETURN_DELIMS);
59
60  EXPECT_TRUE(t.GetNext());
61  EXPECT_EQ(string("this"), t.token());
62
63  EXPECT_TRUE(t.GetNext());
64  EXPECT_EQ(string(" "), t.token());
65
66  EXPECT_TRUE(t.GetNext());
67  EXPECT_EQ(string("is"), t.token());
68
69  EXPECT_TRUE(t.GetNext());
70  EXPECT_EQ(string(" "), t.token());
71
72  EXPECT_TRUE(t.GetNext());
73  EXPECT_EQ(string("a"), t.token());
74
75  EXPECT_TRUE(t.GetNext());
76  EXPECT_EQ(string(" "), t.token());
77
78  EXPECT_TRUE(t.GetNext());
79  EXPECT_EQ(string("test"), t.token());
80
81  EXPECT_FALSE(t.GetNext());
82}
83
84TEST(StringTokenizerTest, ManyDelims) {
85  string input = "this: is, a-test";
86  StringTokenizer t(input, ": ,-");
87
88  EXPECT_TRUE(t.GetNext());
89  EXPECT_EQ(string("this"), t.token());
90
91  EXPECT_TRUE(t.GetNext());
92  EXPECT_EQ(string("is"), t.token());
93
94  EXPECT_TRUE(t.GetNext());
95  EXPECT_EQ(string("a"), t.token());
96
97  EXPECT_TRUE(t.GetNext());
98  EXPECT_EQ(string("test"), t.token());
99
100  EXPECT_FALSE(t.GetNext());
101}
102
103TEST(StringTokenizerTest, ParseHeader) {
104  string input = "Content-Type: text/html ; charset=UTF-8";
105  StringTokenizer t(input, ": ;=");
106  t.set_options(StringTokenizer::RETURN_DELIMS);
107
108  EXPECT_TRUE(t.GetNext());
109  EXPECT_FALSE(t.token_is_delim());
110  EXPECT_EQ(string("Content-Type"), t.token());
111
112  EXPECT_TRUE(t.GetNext());
113  EXPECT_TRUE(t.token_is_delim());
114  EXPECT_EQ(string(":"), t.token());
115
116  EXPECT_TRUE(t.GetNext());
117  EXPECT_TRUE(t.token_is_delim());
118  EXPECT_EQ(string(" "), t.token());
119
120  EXPECT_TRUE(t.GetNext());
121  EXPECT_FALSE(t.token_is_delim());
122  EXPECT_EQ(string("text/html"), t.token());
123
124  EXPECT_TRUE(t.GetNext());
125  EXPECT_TRUE(t.token_is_delim());
126  EXPECT_EQ(string(" "), t.token());
127
128  EXPECT_TRUE(t.GetNext());
129  EXPECT_TRUE(t.token_is_delim());
130  EXPECT_EQ(string(";"), t.token());
131
132  EXPECT_TRUE(t.GetNext());
133  EXPECT_TRUE(t.token_is_delim());
134  EXPECT_EQ(string(" "), t.token());
135
136  EXPECT_TRUE(t.GetNext());
137  EXPECT_FALSE(t.token_is_delim());
138  EXPECT_EQ(string("charset"), t.token());
139
140  EXPECT_TRUE(t.GetNext());
141  EXPECT_TRUE(t.token_is_delim());
142  EXPECT_EQ(string("="), t.token());
143
144  EXPECT_TRUE(t.GetNext());
145  EXPECT_FALSE(t.token_is_delim());
146  EXPECT_EQ(string("UTF-8"), t.token());
147
148  EXPECT_FALSE(t.GetNext());
149  EXPECT_FALSE(t.token_is_delim());
150}
151
152TEST(StringTokenizerTest, ParseQuotedString) {
153  string input = "foo bar 'hello world' baz";
154  StringTokenizer t(input, " ");
155  t.set_quote_chars("'");
156
157  EXPECT_TRUE(t.GetNext());
158  EXPECT_EQ(string("foo"), t.token());
159
160  EXPECT_TRUE(t.GetNext());
161  EXPECT_EQ(string("bar"), t.token());
162
163  EXPECT_TRUE(t.GetNext());
164  EXPECT_EQ(string("'hello world'"), t.token());
165
166  EXPECT_TRUE(t.GetNext());
167  EXPECT_EQ(string("baz"), t.token());
168
169  EXPECT_FALSE(t.GetNext());
170}
171
172TEST(StringTokenizerTest, ParseQuotedString_Malformed) {
173  string input = "bar 'hello wo";
174  StringTokenizer t(input, " ");
175  t.set_quote_chars("'");
176
177  EXPECT_TRUE(t.GetNext());
178  EXPECT_EQ(string("bar"), t.token());
179
180  EXPECT_TRUE(t.GetNext());
181  EXPECT_EQ(string("'hello wo"), t.token());
182
183  EXPECT_FALSE(t.GetNext());
184}
185
186TEST(StringTokenizerTest, ParseQuotedString_Multiple) {
187  string input = "bar 'hel\"lo\" wo' baz\"";
188  StringTokenizer t(input, " ");
189  t.set_quote_chars("'\"");
190
191  EXPECT_TRUE(t.GetNext());
192  EXPECT_EQ(string("bar"), t.token());
193
194  EXPECT_TRUE(t.GetNext());
195  EXPECT_EQ(string("'hel\"lo\" wo'"), t.token());
196
197  EXPECT_TRUE(t.GetNext());
198  EXPECT_EQ(string("baz\""), t.token());
199
200  EXPECT_FALSE(t.GetNext());
201}
202
203TEST(StringTokenizerTest, ParseQuotedString_EscapedQuotes) {
204  string input = "foo 'don\\'t do that'";
205  StringTokenizer t(input, " ");
206  t.set_quote_chars("'");
207
208  EXPECT_TRUE(t.GetNext());
209  EXPECT_EQ(string("foo"), t.token());
210
211  EXPECT_TRUE(t.GetNext());
212  EXPECT_EQ(string("'don\\'t do that'"), t.token());
213
214  EXPECT_FALSE(t.GetNext());
215}
216
217TEST(StringTokenizerTest, ParseQuotedString_EscapedQuotes2) {
218  string input = "foo='a, b', bar";
219  StringTokenizer t(input, ", ");
220  t.set_quote_chars("'");
221
222  EXPECT_TRUE(t.GetNext());
223  EXPECT_EQ(string("foo='a, b'"), t.token());
224
225  EXPECT_TRUE(t.GetNext());
226  EXPECT_EQ(string("bar"), t.token());
227
228  EXPECT_FALSE(t.GetNext());
229}
230