1a3477c862a5debcac7dfb076749059406ec59512Jeff Brown/* 2a3477c862a5debcac7dfb076749059406ec59512Jeff Brown * Copyright (C) 2010 The Android Open Source Project 3a3477c862a5debcac7dfb076749059406ec59512Jeff Brown * 4a3477c862a5debcac7dfb076749059406ec59512Jeff Brown * Licensed under the Apache License, Version 2.0 (the "License"); 5a3477c862a5debcac7dfb076749059406ec59512Jeff Brown * you may not use this file except in compliance with the License. 6a3477c862a5debcac7dfb076749059406ec59512Jeff Brown * You may obtain a copy of the License at 7a3477c862a5debcac7dfb076749059406ec59512Jeff Brown * 8a3477c862a5debcac7dfb076749059406ec59512Jeff Brown * http://www.apache.org/licenses/LICENSE-2.0 9a3477c862a5debcac7dfb076749059406ec59512Jeff Brown * 10a3477c862a5debcac7dfb076749059406ec59512Jeff Brown * Unless required by applicable law or agreed to in writing, software 11a3477c862a5debcac7dfb076749059406ec59512Jeff Brown * distributed under the License is distributed on an "AS IS" BASIS, 12a3477c862a5debcac7dfb076749059406ec59512Jeff Brown * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13a3477c862a5debcac7dfb076749059406ec59512Jeff Brown * See the License for the specific language governing permissions and 14a3477c862a5debcac7dfb076749059406ec59512Jeff Brown * limitations under the License. 15a3477c862a5debcac7dfb076749059406ec59512Jeff Brown */ 16a3477c862a5debcac7dfb076749059406ec59512Jeff Brown 17a3477c862a5debcac7dfb076749059406ec59512Jeff Brown#define LOG_TAG "Tokenizer" 18a3477c862a5debcac7dfb076749059406ec59512Jeff Brown 19a3477c862a5debcac7dfb076749059406ec59512Jeff Brown#include <stdlib.h> 20a3477c862a5debcac7dfb076749059406ec59512Jeff Brown#include <unistd.h> 21a3477c862a5debcac7dfb076749059406ec59512Jeff Brown#include <fcntl.h> 22a3477c862a5debcac7dfb076749059406ec59512Jeff Brown#include <errno.h> 23a3477c862a5debcac7dfb076749059406ec59512Jeff Brown#include <sys/types.h> 24a3477c862a5debcac7dfb076749059406ec59512Jeff Brown#include <sys/stat.h> 25a3477c862a5debcac7dfb076749059406ec59512Jeff Brown#include <utils/Log.h> 26a3477c862a5debcac7dfb076749059406ec59512Jeff Brown#include <utils/Tokenizer.h> 27a3477c862a5debcac7dfb076749059406ec59512Jeff Brown 28a3477c862a5debcac7dfb076749059406ec59512Jeff Brown// Enables debug output for the tokenizer. 29a3477c862a5debcac7dfb076749059406ec59512Jeff Brown#define DEBUG_TOKENIZER 0 30a3477c862a5debcac7dfb076749059406ec59512Jeff Brown 31a3477c862a5debcac7dfb076749059406ec59512Jeff Brown 32a3477c862a5debcac7dfb076749059406ec59512Jeff Brownnamespace android { 33a3477c862a5debcac7dfb076749059406ec59512Jeff Brown 34a3477c862a5debcac7dfb076749059406ec59512Jeff Brownstatic inline bool isDelimiter(char ch, const char* delimiters) { 35a3477c862a5debcac7dfb076749059406ec59512Jeff Brown return strchr(delimiters, ch) != NULL; 36a3477c862a5debcac7dfb076749059406ec59512Jeff Brown} 37a3477c862a5debcac7dfb076749059406ec59512Jeff Brown 38a8be8fa0966521afe78324351e805b4a8351dbd9Jeff BrownTokenizer::Tokenizer(const String8& filename, FileMap* fileMap, char* buffer, 39a8be8fa0966521afe78324351e805b4a8351dbd9Jeff Brown bool ownBuffer, size_t length) : 40db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown mFilename(filename), mFileMap(fileMap), 41a8be8fa0966521afe78324351e805b4a8351dbd9Jeff Brown mBuffer(buffer), mOwnBuffer(ownBuffer), mLength(length), 42a8be8fa0966521afe78324351e805b4a8351dbd9Jeff Brown mCurrent(buffer), mLineNumber(1) { 43a3477c862a5debcac7dfb076749059406ec59512Jeff Brown} 44a3477c862a5debcac7dfb076749059406ec59512Jeff Brown 45a3477c862a5debcac7dfb076749059406ec59512Jeff BrownTokenizer::~Tokenizer() { 469ee93d18edb42d55441b636aa7e001260f1b758dJeff Brown if (mFileMap) { 479ee93d18edb42d55441b636aa7e001260f1b758dJeff Brown mFileMap->release(); 48a8be8fa0966521afe78324351e805b4a8351dbd9Jeff Brown } 49a8be8fa0966521afe78324351e805b4a8351dbd9Jeff Brown if (mOwnBuffer) { 50db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown delete[] mBuffer; 519ee93d18edb42d55441b636aa7e001260f1b758dJeff Brown } 52a3477c862a5debcac7dfb076749059406ec59512Jeff Brown} 53a3477c862a5debcac7dfb076749059406ec59512Jeff Brown 54a3477c862a5debcac7dfb076749059406ec59512Jeff Brownstatus_t Tokenizer::open(const String8& filename, Tokenizer** outTokenizer) { 55a3477c862a5debcac7dfb076749059406ec59512Jeff Brown *outTokenizer = NULL; 56a3477c862a5debcac7dfb076749059406ec59512Jeff Brown 57a3477c862a5debcac7dfb076749059406ec59512Jeff Brown int result = NO_ERROR; 58a3477c862a5debcac7dfb076749059406ec59512Jeff Brown int fd = ::open(filename.string(), O_RDONLY); 59a3477c862a5debcac7dfb076749059406ec59512Jeff Brown if (fd < 0) { 60a3477c862a5debcac7dfb076749059406ec59512Jeff Brown result = -errno; 61e6f43ddce78d6846af12550ff9193c5c6fe5844bSteve Block ALOGE("Error opening file '%s', %s.", filename.string(), strerror(errno)); 62a3477c862a5debcac7dfb076749059406ec59512Jeff Brown } else { 639ee93d18edb42d55441b636aa7e001260f1b758dJeff Brown struct stat stat; 649ee93d18edb42d55441b636aa7e001260f1b758dJeff Brown if (fstat(fd, &stat)) { 65a3477c862a5debcac7dfb076749059406ec59512Jeff Brown result = -errno; 66e6f43ddce78d6846af12550ff9193c5c6fe5844bSteve Block ALOGE("Error getting size of file '%s', %s.", filename.string(), strerror(errno)); 67a3477c862a5debcac7dfb076749059406ec59512Jeff Brown } else { 68a3477c862a5debcac7dfb076749059406ec59512Jeff Brown size_t length = size_t(stat.st_size); 69db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown 709ee93d18edb42d55441b636aa7e001260f1b758dJeff Brown FileMap* fileMap = new FileMap(); 71a8be8fa0966521afe78324351e805b4a8351dbd9Jeff Brown bool ownBuffer = false; 72db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown char* buffer; 73db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown if (fileMap->create(NULL, fd, 0, length, true)) { 749ee93d18edb42d55441b636aa7e001260f1b758dJeff Brown fileMap->advise(FileMap::SEQUENTIAL); 75db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown buffer = static_cast<char*>(fileMap->getDataPtr()); 76db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown } else { 77db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown fileMap->release(); 78db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown fileMap = NULL; 79a3477c862a5debcac7dfb076749059406ec59512Jeff Brown 80db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown // Fall back to reading into a buffer since we can't mmap files in sysfs. 81db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown // The length we obtained from stat is wrong too (it will always be 4096) 82db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown // so we must trust that read will read the entire file. 83db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown buffer = new char[length]; 84a8be8fa0966521afe78324351e805b4a8351dbd9Jeff Brown ownBuffer = true; 85db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown ssize_t nrd = read(fd, buffer, length); 86db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown if (nrd < 0) { 87db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown result = -errno; 88e6f43ddce78d6846af12550ff9193c5c6fe5844bSteve Block ALOGE("Error reading file '%s', %s.", filename.string(), strerror(errno)); 89db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown delete[] buffer; 90db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown buffer = NULL; 91db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown } else { 92db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown length = size_t(nrd); 93a3477c862a5debcac7dfb076749059406ec59512Jeff Brown } 94a3477c862a5debcac7dfb076749059406ec59512Jeff Brown } 95db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown 96db360642ed7a48eb3b3607a791bbe449cc6529bbJeff Brown if (!result) { 97a8be8fa0966521afe78324351e805b4a8351dbd9Jeff Brown *outTokenizer = new Tokenizer(filename, fileMap, buffer, ownBuffer, length); 989ee93d18edb42d55441b636aa7e001260f1b758dJeff Brown } 99a3477c862a5debcac7dfb076749059406ec59512Jeff Brown } 100a3477c862a5debcac7dfb076749059406ec59512Jeff Brown close(fd); 101a3477c862a5debcac7dfb076749059406ec59512Jeff Brown } 102a3477c862a5debcac7dfb076749059406ec59512Jeff Brown return result; 103a3477c862a5debcac7dfb076749059406ec59512Jeff Brown} 104a3477c862a5debcac7dfb076749059406ec59512Jeff Brown 105a8be8fa0966521afe78324351e805b4a8351dbd9Jeff Brownstatus_t Tokenizer::fromContents(const String8& filename, 106a8be8fa0966521afe78324351e805b4a8351dbd9Jeff Brown const char* contents, Tokenizer** outTokenizer) { 107a8be8fa0966521afe78324351e805b4a8351dbd9Jeff Brown *outTokenizer = new Tokenizer(filename, NULL, 108a8be8fa0966521afe78324351e805b4a8351dbd9Jeff Brown const_cast<char*>(contents), false, strlen(contents)); 109a8be8fa0966521afe78324351e805b4a8351dbd9Jeff Brown return OK; 110a8be8fa0966521afe78324351e805b4a8351dbd9Jeff Brown} 111a8be8fa0966521afe78324351e805b4a8351dbd9Jeff Brown 112a3477c862a5debcac7dfb076749059406ec59512Jeff BrownString8 Tokenizer::getLocation() const { 113a3477c862a5debcac7dfb076749059406ec59512Jeff Brown String8 result; 114a3477c862a5debcac7dfb076749059406ec59512Jeff Brown result.appendFormat("%s:%d", mFilename.string(), mLineNumber); 115a3477c862a5debcac7dfb076749059406ec59512Jeff Brown return result; 116a3477c862a5debcac7dfb076749059406ec59512Jeff Brown} 117a3477c862a5debcac7dfb076749059406ec59512Jeff Brown 118a3477c862a5debcac7dfb076749059406ec59512Jeff BrownString8 Tokenizer::peekRemainderOfLine() const { 119a3477c862a5debcac7dfb076749059406ec59512Jeff Brown const char* end = getEnd(); 120a3477c862a5debcac7dfb076749059406ec59512Jeff Brown const char* eol = mCurrent; 121a3477c862a5debcac7dfb076749059406ec59512Jeff Brown while (eol != end) { 122a3477c862a5debcac7dfb076749059406ec59512Jeff Brown char ch = *eol; 123a3477c862a5debcac7dfb076749059406ec59512Jeff Brown if (ch == '\n') { 124a3477c862a5debcac7dfb076749059406ec59512Jeff Brown break; 125a3477c862a5debcac7dfb076749059406ec59512Jeff Brown } 126a3477c862a5debcac7dfb076749059406ec59512Jeff Brown eol += 1; 127a3477c862a5debcac7dfb076749059406ec59512Jeff Brown } 128a3477c862a5debcac7dfb076749059406ec59512Jeff Brown return String8(mCurrent, eol - mCurrent); 129a3477c862a5debcac7dfb076749059406ec59512Jeff Brown} 130a3477c862a5debcac7dfb076749059406ec59512Jeff Brown 131a3477c862a5debcac7dfb076749059406ec59512Jeff BrownString8 Tokenizer::nextToken(const char* delimiters) { 132a3477c862a5debcac7dfb076749059406ec59512Jeff Brown#if DEBUG_TOKENIZER 1339d4536835248525f32f1504a3d28d5bbfa0a2910Steve Block ALOGD("nextToken"); 134a3477c862a5debcac7dfb076749059406ec59512Jeff Brown#endif 135a3477c862a5debcac7dfb076749059406ec59512Jeff Brown const char* end = getEnd(); 136a3477c862a5debcac7dfb076749059406ec59512Jeff Brown const char* tokenStart = mCurrent; 137a3477c862a5debcac7dfb076749059406ec59512Jeff Brown while (mCurrent != end) { 138a3477c862a5debcac7dfb076749059406ec59512Jeff Brown char ch = *mCurrent; 139a3477c862a5debcac7dfb076749059406ec59512Jeff Brown if (ch == '\n' || isDelimiter(ch, delimiters)) { 140a3477c862a5debcac7dfb076749059406ec59512Jeff Brown break; 141a3477c862a5debcac7dfb076749059406ec59512Jeff Brown } 142a3477c862a5debcac7dfb076749059406ec59512Jeff Brown mCurrent += 1; 143a3477c862a5debcac7dfb076749059406ec59512Jeff Brown } 144a3477c862a5debcac7dfb076749059406ec59512Jeff Brown return String8(tokenStart, mCurrent - tokenStart); 145a3477c862a5debcac7dfb076749059406ec59512Jeff Brown} 146a3477c862a5debcac7dfb076749059406ec59512Jeff Brown 147a3477c862a5debcac7dfb076749059406ec59512Jeff Brownvoid Tokenizer::nextLine() { 148a3477c862a5debcac7dfb076749059406ec59512Jeff Brown#if DEBUG_TOKENIZER 1499d4536835248525f32f1504a3d28d5bbfa0a2910Steve Block ALOGD("nextLine"); 150a3477c862a5debcac7dfb076749059406ec59512Jeff Brown#endif 151a3477c862a5debcac7dfb076749059406ec59512Jeff Brown const char* end = getEnd(); 152a3477c862a5debcac7dfb076749059406ec59512Jeff Brown while (mCurrent != end) { 153a3477c862a5debcac7dfb076749059406ec59512Jeff Brown char ch = *(mCurrent++); 154a3477c862a5debcac7dfb076749059406ec59512Jeff Brown if (ch == '\n') { 155a3477c862a5debcac7dfb076749059406ec59512Jeff Brown mLineNumber += 1; 156a3477c862a5debcac7dfb076749059406ec59512Jeff Brown break; 157a3477c862a5debcac7dfb076749059406ec59512Jeff Brown } 158a3477c862a5debcac7dfb076749059406ec59512Jeff Brown } 159a3477c862a5debcac7dfb076749059406ec59512Jeff Brown} 160a3477c862a5debcac7dfb076749059406ec59512Jeff Brown 161a3477c862a5debcac7dfb076749059406ec59512Jeff Brownvoid Tokenizer::skipDelimiters(const char* delimiters) { 162a3477c862a5debcac7dfb076749059406ec59512Jeff Brown#if DEBUG_TOKENIZER 1639d4536835248525f32f1504a3d28d5bbfa0a2910Steve Block ALOGD("skipDelimiters"); 164a3477c862a5debcac7dfb076749059406ec59512Jeff Brown#endif 165a3477c862a5debcac7dfb076749059406ec59512Jeff Brown const char* end = getEnd(); 166a3477c862a5debcac7dfb076749059406ec59512Jeff Brown while (mCurrent != end) { 167a3477c862a5debcac7dfb076749059406ec59512Jeff Brown char ch = *mCurrent; 168a3477c862a5debcac7dfb076749059406ec59512Jeff Brown if (ch == '\n' || !isDelimiter(ch, delimiters)) { 169a3477c862a5debcac7dfb076749059406ec59512Jeff Brown break; 170a3477c862a5debcac7dfb076749059406ec59512Jeff Brown } 171a3477c862a5debcac7dfb076749059406ec59512Jeff Brown mCurrent += 1; 172a3477c862a5debcac7dfb076749059406ec59512Jeff Brown } 173a3477c862a5debcac7dfb076749059406ec59512Jeff Brown} 174a3477c862a5debcac7dfb076749059406ec59512Jeff Brown 175a3477c862a5debcac7dfb076749059406ec59512Jeff Brown} // namespace android 176