1647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown/* 2647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown * Copyright (C) 2010 The Android Open Source Project 3647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown * 4647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown * Licensed under the Apache License, Version 2.0 (the "License"); 5647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown * you may not use this file except in compliance with the License. 6647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown * You may obtain a copy of the License at 7647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown * 8647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown * http://www.apache.org/licenses/LICENSE-2.0 9647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown * 10647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown * Unless required by applicable law or agreed to in writing, software 11647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown * distributed under the License is distributed on an "AS IS" BASIS, 12647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown * See the License for the specific language governing permissions and 14647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown * limitations under the License. 15647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown */ 16647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown 17647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown#define LOG_TAG "Tokenizer" 18647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown 19647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown#include <stdlib.h> 20647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown#include <unistd.h> 21647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown#include <fcntl.h> 22647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown#include <errno.h> 23647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown#include <sys/types.h> 24647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown#include <sys/stat.h> 25647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown#include <utils/Log.h> 26647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown#include <utils/Tokenizer.h> 27647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown 28647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown// Enables debug output for the tokenizer. 29647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown#define DEBUG_TOKENIZER 0 30647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown 31647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown 32647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brownnamespace android { 33647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown 34647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brownstatic inline bool isDelimiter(char ch, const char* delimiters) { 35647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown return strchr(delimiters, ch) != NULL; 36647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown} 37647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown 382c1627dc49994f83a636efd1970825b519bd93cbJeff BrownTokenizer::Tokenizer(const String8& filename, FileMap* fileMap, char* buffer, 392c1627dc49994f83a636efd1970825b519bd93cbJeff Brown bool ownBuffer, size_t length) : 401d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown mFilename(filename), mFileMap(fileMap), 412c1627dc49994f83a636efd1970825b519bd93cbJeff Brown mBuffer(buffer), mOwnBuffer(ownBuffer), mLength(length), 422c1627dc49994f83a636efd1970825b519bd93cbJeff Brown mCurrent(buffer), mLineNumber(1) { 43647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown} 44647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown 45647925ddf053989b641b4c5c8a51efd55c931f22Jeff BrownTokenizer::~Tokenizer() { 466832a7a4e04c84192f0bafc6ac990fc79cd13882Narayan Kamath delete mFileMap; 472c1627dc49994f83a636efd1970825b519bd93cbJeff Brown if (mOwnBuffer) { 481d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown delete[] mBuffer; 49d36ec3afdac0ca4158a786b96599ee8bdf64b043Jeff Brown } 50647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown} 51647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown 52647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brownstatus_t Tokenizer::open(const String8& filename, Tokenizer** outTokenizer) { 53647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown *outTokenizer = NULL; 54647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown 55647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown int result = NO_ERROR; 56647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown int fd = ::open(filename.string(), O_RDONLY); 57647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown if (fd < 0) { 58647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown result = -errno; 596ed68cc412752e4c78755df9a1516e610ec66fa8Elliott Hughes ALOGE("Error opening file '%s': %s", filename.string(), strerror(errno)); 60647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown } else { 61d36ec3afdac0ca4158a786b96599ee8bdf64b043Jeff Brown struct stat stat; 62d36ec3afdac0ca4158a786b96599ee8bdf64b043Jeff Brown if (fstat(fd, &stat)) { 63647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown result = -errno; 646ed68cc412752e4c78755df9a1516e610ec66fa8Elliott Hughes ALOGE("Error getting size of file '%s': %s", filename.string(), strerror(errno)); 65647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown } else { 66647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown size_t length = size_t(stat.st_size); 671d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown 68d36ec3afdac0ca4158a786b96599ee8bdf64b043Jeff Brown FileMap* fileMap = new FileMap(); 692c1627dc49994f83a636efd1970825b519bd93cbJeff Brown bool ownBuffer = false; 701d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown char* buffer; 711d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown if (fileMap->create(NULL, fd, 0, length, true)) { 72d36ec3afdac0ca4158a786b96599ee8bdf64b043Jeff Brown fileMap->advise(FileMap::SEQUENTIAL); 731d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown buffer = static_cast<char*>(fileMap->getDataPtr()); 741d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown } else { 756832a7a4e04c84192f0bafc6ac990fc79cd13882Narayan Kamath delete fileMap; 761d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown fileMap = NULL; 77647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown 781d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown // Fall back to reading into a buffer since we can't mmap files in sysfs. 791d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown // The length we obtained from stat is wrong too (it will always be 4096) 801d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown // so we must trust that read will read the entire file. 811d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown buffer = new char[length]; 822c1627dc49994f83a636efd1970825b519bd93cbJeff Brown ownBuffer = true; 831d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown ssize_t nrd = read(fd, buffer, length); 841d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown if (nrd < 0) { 851d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown result = -errno; 866ed68cc412752e4c78755df9a1516e610ec66fa8Elliott Hughes ALOGE("Error reading file '%s': %s", filename.string(), strerror(errno)); 871d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown delete[] buffer; 881d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown buffer = NULL; 891d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown } else { 901d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown length = size_t(nrd); 91647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown } 92647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown } 931d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown 941d618d63c1bb99728b5b0afe320f5a6afa95436cJeff Brown if (!result) { 952c1627dc49994f83a636efd1970825b519bd93cbJeff Brown *outTokenizer = new Tokenizer(filename, fileMap, buffer, ownBuffer, length); 96d36ec3afdac0ca4158a786b96599ee8bdf64b043Jeff Brown } 97647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown } 98647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown close(fd); 99647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown } 100647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown return result; 101647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown} 102647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown 1032c1627dc49994f83a636efd1970825b519bd93cbJeff Brownstatus_t Tokenizer::fromContents(const String8& filename, 1042c1627dc49994f83a636efd1970825b519bd93cbJeff Brown const char* contents, Tokenizer** outTokenizer) { 1052c1627dc49994f83a636efd1970825b519bd93cbJeff Brown *outTokenizer = new Tokenizer(filename, NULL, 1062c1627dc49994f83a636efd1970825b519bd93cbJeff Brown const_cast<char*>(contents), false, strlen(contents)); 1072c1627dc49994f83a636efd1970825b519bd93cbJeff Brown return OK; 1082c1627dc49994f83a636efd1970825b519bd93cbJeff Brown} 1092c1627dc49994f83a636efd1970825b519bd93cbJeff Brown 110647925ddf053989b641b4c5c8a51efd55c931f22Jeff BrownString8 Tokenizer::getLocation() const { 111647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown String8 result; 112647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown result.appendFormat("%s:%d", mFilename.string(), mLineNumber); 113647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown return result; 114647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown} 115647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown 116647925ddf053989b641b4c5c8a51efd55c931f22Jeff BrownString8 Tokenizer::peekRemainderOfLine() const { 117647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown const char* end = getEnd(); 118647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown const char* eol = mCurrent; 119647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown while (eol != end) { 120647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown char ch = *eol; 121647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown if (ch == '\n') { 122647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown break; 123647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown } 124647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown eol += 1; 125647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown } 126647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown return String8(mCurrent, eol - mCurrent); 127647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown} 128647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown 129647925ddf053989b641b4c5c8a51efd55c931f22Jeff BrownString8 Tokenizer::nextToken(const char* delimiters) { 130647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown#if DEBUG_TOKENIZER 131eb0953307ce75cec031aedbf21abff08e5a737e5Steve Block ALOGD("nextToken"); 132647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown#endif 133647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown const char* end = getEnd(); 134647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown const char* tokenStart = mCurrent; 135647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown while (mCurrent != end) { 136647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown char ch = *mCurrent; 137647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown if (ch == '\n' || isDelimiter(ch, delimiters)) { 138647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown break; 139647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown } 140647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown mCurrent += 1; 141647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown } 142647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown return String8(tokenStart, mCurrent - tokenStart); 143647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown} 144647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown 145647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brownvoid Tokenizer::nextLine() { 146647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown#if DEBUG_TOKENIZER 147eb0953307ce75cec031aedbf21abff08e5a737e5Steve Block ALOGD("nextLine"); 148647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown#endif 149647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown const char* end = getEnd(); 150647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown while (mCurrent != end) { 151647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown char ch = *(mCurrent++); 152647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown if (ch == '\n') { 153647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown mLineNumber += 1; 154647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown break; 155647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown } 156647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown } 157647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown} 158647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown 159647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brownvoid Tokenizer::skipDelimiters(const char* delimiters) { 160647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown#if DEBUG_TOKENIZER 161eb0953307ce75cec031aedbf21abff08e5a737e5Steve Block ALOGD("skipDelimiters"); 162647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown#endif 163647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown const char* end = getEnd(); 164647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown while (mCurrent != end) { 165647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown char ch = *mCurrent; 166647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown if (ch == '\n' || !isDelimiter(ch, delimiters)) { 167647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown break; 168647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown } 169647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown mCurrent += 1; 170647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown } 171647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown} 172647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown 173647925ddf053989b641b4c5c8a51efd55c931f22Jeff Brown} // namespace android 174