1bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi/* 2bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi * Copyright (C) 2017 The Android Open Source Project 3bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi * 4bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi * Licensed under the Apache License, Version 2.0 (the "License"); 5bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi * you may not use this file except in compliance with the License. 6bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi * You may obtain a copy of the License at 7bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi * 8bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi * http://www.apache.org/licenses/LICENSE-2.0 9bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi * 10bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi * Unless required by applicable law or agreed to in writing, software 11bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi * distributed under the License is distributed on an "AS IS" BASIS, 12bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi * See the License for the specific language governing permissions and 14bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi * limitations under the License. 15bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi */ 16bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi 17bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi#ifndef LIBTEXTCLASSIFIER_SMARTSELECT_TOKENIZER_H_ 18bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi#define LIBTEXTCLASSIFIER_SMARTSELECT_TOKENIZER_H_ 19bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi 20bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi#include <string> 21bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi#include <vector> 22bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi 23bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi#include "smartselect/tokenizer.pb.h" 24bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi#include "smartselect/types.h" 25bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi#include "util/base/integral_types.h" 26bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi 27bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifinamespace libtextclassifier { 28bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi 29bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi// Tokenizer splits the input string into a sequence of tokens, according to the 30bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi// configuration. 31bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharificlass Tokenizer { 32bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi public: 33bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi explicit Tokenizer( 34bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi const std::vector<TokenizationCodepointRange>& codepoint_range_configs) { 35bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi PrepareTokenizationCodepointRanges(codepoint_range_configs); 36bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi } 37bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi 38bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi // Tokenizes the input string using the selected tokenization method. 39bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi std::vector<Token> Tokenize(const std::string& utf8_text) const; 40bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi 41bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi protected: 4226e8c2edb55f8c4efc47bc586edd5f9506e281a7Lukas Zilka // Represents a codepoint range [start, end) with its role for tokenization. 4326e8c2edb55f8c4efc47bc586edd5f9506e281a7Lukas Zilka struct CodepointRange { 4426e8c2edb55f8c4efc47bc586edd5f9506e281a7Lukas Zilka int32 start; 4526e8c2edb55f8c4efc47bc586edd5f9506e281a7Lukas Zilka int32 end; 4626e8c2edb55f8c4efc47bc586edd5f9506e281a7Lukas Zilka TokenizationCodepointRange::Role role; 4726e8c2edb55f8c4efc47bc586edd5f9506e281a7Lukas Zilka 4826e8c2edb55f8c4efc47bc586edd5f9506e281a7Lukas Zilka CodepointRange(int32 arg_start, int32 arg_end, 4926e8c2edb55f8c4efc47bc586edd5f9506e281a7Lukas Zilka TokenizationCodepointRange::Role arg_role) 5026e8c2edb55f8c4efc47bc586edd5f9506e281a7Lukas Zilka : start(arg_start), end(arg_end), role(arg_role) {} 5126e8c2edb55f8c4efc47bc586edd5f9506e281a7Lukas Zilka }; 5226e8c2edb55f8c4efc47bc586edd5f9506e281a7Lukas Zilka 53bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi // Prepares tokenization codepoint ranges for use in tokenization. 54bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi void PrepareTokenizationCodepointRanges( 5526e8c2edb55f8c4efc47bc586edd5f9506e281a7Lukas Zilka const std::vector<TokenizationCodepointRange>& codepoint_range_configs); 56bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi 57bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi // Finds the tokenization role for given codepoint. 58bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi // If the character is not found returns DEFAULT_ROLE. 5991f068673f5915dd4c12fac026e4a043e4a063a0Matt Sharifi // Internally uses binary search so should be O(log(# of codepoint_ranges)). 60bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi TokenizationCodepointRange::Role FindTokenizationRole(int codepoint) const; 61bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi 62bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi private: 63bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi // Codepoint ranges that determine how different codepoints are tokenized. 64bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi // The ranges must not overlap. 65bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi std::vector<CodepointRange> codepoint_ranges_; 66bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi}; 67bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi 68bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi} // namespace libtextclassifier 69bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi 70bda09f1da39ce38a5ece4757b82a64776e53214cMatt Sharifi#endif // LIBTEXTCLASSIFIER_SMARTSELECT_TOKENIZER_H_ 71