199b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner/* 299b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * Copyright (c) 2015, Intel Corporation 399b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * All rights reserved. 499b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * 599b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * Redistribution and use in source and binary forms, with or without modification, 699b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * are permitted provided that the following conditions are met: 799b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * 899b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * 1. Redistributions of source code must retain the above copyright notice, this 999b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * list of conditions and the following disclaimer. 1099b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * 1199b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * 2. Redistributions in binary form must reproduce the above copyright notice, 1299b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * this list of conditions and the following disclaimer in the documentation and/or 1399b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * other materials provided with the distribution. 1499b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * 1599b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * 3. Neither the name of the copyright holder nor the names of its contributors 1699b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * may be used to endorse or promote products derived from this software without 1799b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * specific prior written permission. 1899b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * 1999b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 2099b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 2199b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 2299b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 2399b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 2499b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 2599b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 2699b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2799b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 2899b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2999b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner */ 3099b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner#include "Tokenizer.h" 3199b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner 3299b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagnerusing std::string; 3399b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagnerusing std::vector; 3499b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner 3599b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagnerconst string Tokenizer::defaultDelimiters = " \n\r\t\v\f"; 3699b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner 379368eea42a1afb01dd44110582f997115b50e742François GaffieTokenizer::Tokenizer(const string &input, const string &delimiters, bool mergeDelimiters) 389368eea42a1afb01dd44110582f997115b50e742François Gaffie : _input(input), _delimiters(delimiters), _mergeDelimiters(mergeDelimiters) 3999b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner{ 4099b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner} 4199b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner 4299b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagnervector<string> Tokenizer::split() 4399b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner{ 4499b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner vector<string> result; 4599b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner string token; 469368eea42a1afb01dd44110582f997115b50e742François Gaffie bool leftover = false; 479368eea42a1afb01dd44110582f997115b50e742François Gaffie 489368eea42a1afb01dd44110582f997115b50e742François Gaffie for (const auto character : _input) { 499368eea42a1afb01dd44110582f997115b50e742François Gaffie if (_delimiters.find(character) != string::npos) { 509368eea42a1afb01dd44110582f997115b50e742François Gaffie if (_mergeDelimiters) { 519368eea42a1afb01dd44110582f997115b50e742François Gaffie leftover = false; 529368eea42a1afb01dd44110582f997115b50e742François Gaffie if (token.empty()) { 539368eea42a1afb01dd44110582f997115b50e742François Gaffie // skip consecutive delimiters 549368eea42a1afb01dd44110582f997115b50e742François Gaffie continue; 559368eea42a1afb01dd44110582f997115b50e742François Gaffie } 569368eea42a1afb01dd44110582f997115b50e742François Gaffie } else { 579368eea42a1afb01dd44110582f997115b50e742François Gaffie // We've encountered a delimiter, which means that there is a 589368eea42a1afb01dd44110582f997115b50e742François Gaffie // left-hand token and a right-side token. We are going to add 599368eea42a1afb01dd44110582f997115b50e742François Gaffie // the left-hand one but must not forget that there is a 609368eea42a1afb01dd44110582f997115b50e742François Gaffie // right-hand one (possibly empty) 619368eea42a1afb01dd44110582f997115b50e742François Gaffie leftover = true; 629368eea42a1afb01dd44110582f997115b50e742François Gaffie } 6399b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner 649368eea42a1afb01dd44110582f997115b50e742François Gaffie result.push_back(token); 659368eea42a1afb01dd44110582f997115b50e742François Gaffie token.clear(); 669368eea42a1afb01dd44110582f997115b50e742François Gaffie continue; 6799b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner } 689368eea42a1afb01dd44110582f997115b50e742François Gaffie token += character; 699368eea42a1afb01dd44110582f997115b50e742François Gaffie leftover = true; 709368eea42a1afb01dd44110582f997115b50e742François Gaffie } 719368eea42a1afb01dd44110582f997115b50e742François Gaffie 729368eea42a1afb01dd44110582f997115b50e742François Gaffie // push any leftover token: 739368eea42a1afb01dd44110582f997115b50e742François Gaffie if (leftover) { 7499b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner result.push_back(token); 7599b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner } 769368eea42a1afb01dd44110582f997115b50e742François Gaffie 779368eea42a1afb01dd44110582f997115b50e742François Gaffie return result; 7899b3e852e553103ff75ed38117baa5bce2befbdbDavid Wagner} 79