StatsLogProcessor.cpp revision 69f1baf7dd3d6e595abdae000adf0f197558d0ea
1/* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#define DEBUG true // STOPSHIP if true 18#include "Log.h" 19#include "statslog.h" 20 21#include <android-base/file.h> 22#include <dirent.h> 23#include "StatsLogProcessor.h" 24#include "android-base/stringprintf.h" 25#include "guardrail/StatsdStats.h" 26#include "metrics/CountMetricProducer.h" 27#include "stats_util.h" 28#include "storage/StorageManager.h" 29 30#include <log/log_event_list.h> 31#include <utils/Errors.h> 32 33using namespace android; 34using android::base::StringPrintf; 35using android::util::FIELD_COUNT_REPEATED; 36using android::util::FIELD_TYPE_BOOL; 37using android::util::FIELD_TYPE_FLOAT; 38using android::util::FIELD_TYPE_INT32; 39using android::util::FIELD_TYPE_INT64; 40using android::util::FIELD_TYPE_MESSAGE; 41using android::util::FIELD_TYPE_STRING; 42using android::util::ProtoOutputStream; 43using std::make_unique; 44using std::unique_ptr; 45using std::vector; 46 47namespace android { 48namespace os { 49namespace statsd { 50 51// for ConfigMetricsReportList 52const int FIELD_ID_CONFIG_KEY = 1; 53const int FIELD_ID_REPORTS = 2; 54// for ConfigKey 55const int FIELD_ID_UID = 1; 56const int FIELD_ID_NAME = 2; 57// for ConfigMetricsReport 58const int FIELD_ID_METRICS = 1; 59const int FIELD_ID_UID_MAP = 2; 60 61#define STATS_DATA_DIR "/data/system/stats-data" 62 63StatsLogProcessor::StatsLogProcessor(const sp<UidMap>& uidMap, 64 const sp<AnomalyMonitor>& anomalyMonitor, 65 const std::function<void(const ConfigKey&)>& sendBroadcast) 66 : mUidMap(uidMap), mAnomalyMonitor(anomalyMonitor), mSendBroadcast(sendBroadcast) { 67 // On each initialization of StatsLogProcessor, check stats-data directory to see if there is 68 // any left over data to be read. 69 StorageManager::sendBroadcast(STATS_DATA_DIR, mSendBroadcast); 70} 71 72StatsLogProcessor::~StatsLogProcessor() { 73} 74 75void StatsLogProcessor::onAnomalyAlarmFired( 76 const uint64_t timestampNs, 77 unordered_set<sp<const AnomalyAlarm>, SpHash<AnomalyAlarm>> anomalySet) { 78 for (const auto& anomaly : anomalySet) { 79 for (const auto& itr : mMetricsManagers) { 80 itr.second->onAnomalyAlarmFired(timestampNs, anomaly); 81 } 82 } 83} 84 85// TODO: what if statsd service restarts? How do we know what logs are already processed before? 86void StatsLogProcessor::OnLogEvent(const LogEvent& msg) { 87 StatsdStats::getInstance().noteAtomLogged(msg.GetTagId(), msg.GetTimestampNs() / NS_PER_SEC); 88 // pass the event to metrics managers. 89 for (auto& pair : mMetricsManagers) { 90 pair.second->onLogEvent(msg); 91 flushIfNecessary(msg.GetTimestampNs(), pair.first, pair.second); 92 } 93 94 // Hard-coded logic to update the isolated uid's in the uid-map. 95 // The field numbers need to be currently updated by hand with atoms.proto 96 if (msg.GetTagId() == android::util::ISOLATED_UID_CHANGED) { 97 status_t err = NO_ERROR, err2 = NO_ERROR, err3 = NO_ERROR; 98 bool is_create = msg.GetBool(3, &err); 99 auto parent_uid = int(msg.GetLong(1, &err2)); 100 auto isolated_uid = int(msg.GetLong(2, &err3)); 101 if (err == NO_ERROR && err2 == NO_ERROR && err3 == NO_ERROR) { 102 if (is_create) { 103 mUidMap->assignIsolatedUid(isolated_uid, parent_uid); 104 } else { 105 mUidMap->removeIsolatedUid(isolated_uid, parent_uid); 106 } 107 } 108 } 109} 110 111void StatsLogProcessor::OnConfigUpdated(const ConfigKey& key, const StatsdConfig& config) { 112 ALOGD("Updated configuration for key %s", key.ToString().c_str()); 113 unique_ptr<MetricsManager> newMetricsManager = std::make_unique<MetricsManager>(key, config); 114 115 auto it = mMetricsManagers.find(key); 116 if (it != mMetricsManagers.end()) { 117 it->second->finish(); 118 } else if (mMetricsManagers.size() > StatsdStats::kMaxConfigCount) { 119 ALOGE("Can't accept more configs!"); 120 return; 121 } 122 123 if (newMetricsManager->isConfigValid()) { 124 mUidMap->OnConfigUpdated(key); 125 newMetricsManager->setAnomalyMonitor(mAnomalyMonitor); 126 mMetricsManagers[key] = std::move(newMetricsManager); 127 // Why doesn't this work? mMetricsManagers.insert({key, std::move(newMetricsManager)}); 128 VLOG("StatsdConfig valid"); 129 } else { 130 // If there is any error in the config, don't use it. 131 ALOGE("StatsdConfig NOT valid"); 132 } 133} 134 135size_t StatsLogProcessor::GetMetricsSize(const ConfigKey& key) const { 136 auto it = mMetricsManagers.find(key); 137 if (it == mMetricsManagers.end()) { 138 ALOGW("Config source %s does not exist", key.ToString().c_str()); 139 return 0; 140 } 141 return it->second->byteSize(); 142} 143 144void StatsLogProcessor::onDumpReport(const ConfigKey& key, vector<uint8_t>* outData) { 145 auto it = mMetricsManagers.find(key); 146 if (it == mMetricsManagers.end()) { 147 ALOGW("Config source %s does not exist", key.ToString().c_str()); 148 return; 149 } 150 151 // This allows another broadcast to be sent within the rate-limit period if we get close to 152 // filling the buffer again soon. 153 mBroadcastTimesMutex.lock(); 154 mLastBroadcastTimes.erase(key); 155 mBroadcastTimesMutex.unlock(); 156 157 ProtoOutputStream proto; 158 159 // Start of ConfigKey. 160 long long configKeyToken = proto.start(FIELD_TYPE_MESSAGE | FIELD_ID_CONFIG_KEY); 161 proto.write(FIELD_TYPE_INT32 | FIELD_ID_UID, key.GetUid()); 162 proto.write(FIELD_TYPE_STRING | FIELD_ID_NAME, key.GetName()); 163 proto.end(configKeyToken); 164 // End of ConfigKey. 165 166 // Start of ConfigMetricsReport (reports). 167 long long reportsToken = 168 proto.start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS); 169 170 // First, fill in ConfigMetricsReport using current data on memory, which 171 // starts from filling in StatsLogReport's. 172 for (auto& m : it->second->onDumpReport()) { 173 // Add each vector of StatsLogReport into a repeated field. 174 proto.write(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_METRICS, 175 reinterpret_cast<char*>(m.get()->data()), m.get()->size()); 176 } 177 178 // Fill in UidMap. 179 auto uidMap = mUidMap->getOutput(key); 180 const int uidMapSize = uidMap.ByteSize(); 181 char uidMapBuffer[uidMapSize]; 182 uidMap.SerializeToArray(&uidMapBuffer[0], uidMapSize); 183 proto.write(FIELD_TYPE_MESSAGE | FIELD_ID_UID_MAP, uidMapBuffer, uidMapSize); 184 185 // End of ConfigMetricsReport (reports). 186 proto.end(reportsToken); 187 188 // Then, check stats-data directory to see there's any file containing 189 // ConfigMetricsReport from previous shutdowns to concatenate to reports. 190 StorageManager::appendConfigMetricsReport(STATS_DATA_DIR, proto); 191 192 if (outData != nullptr) { 193 outData->clear(); 194 outData->resize(proto.size()); 195 size_t pos = 0; 196 auto iter = proto.data(); 197 while (iter.readBuffer() != NULL) { 198 size_t toRead = iter.currentToRead(); 199 std::memcpy(&((*outData)[pos]), iter.readBuffer(), toRead); 200 pos += toRead; 201 iter.rp()->move(toRead); 202 } 203 } 204 StatsdStats::getInstance().noteMetricsReportSent(key); 205} 206 207void StatsLogProcessor::OnConfigRemoved(const ConfigKey& key) { 208 auto it = mMetricsManagers.find(key); 209 if (it != mMetricsManagers.end()) { 210 it->second->finish(); 211 mMetricsManagers.erase(it); 212 mUidMap->OnConfigRemoved(key); 213 } 214 StatsdStats::getInstance().noteConfigRemoved(key); 215 216 std::lock_guard<std::mutex> lock(mBroadcastTimesMutex); 217 mLastBroadcastTimes.erase(key); 218} 219 220void StatsLogProcessor::flushIfNecessary(uint64_t timestampNs, 221 const ConfigKey& key, 222 const unique_ptr<MetricsManager>& metricsManager) { 223 std::lock_guard<std::mutex> lock(mBroadcastTimesMutex); 224 225 size_t totalBytes = metricsManager->byteSize(); 226 if (totalBytes > .9 * kMaxSerializedBytes) { // Send broadcast so that receivers can pull data. 227 auto lastFlushNs = mLastBroadcastTimes.find(key); 228 if (lastFlushNs != mLastBroadcastTimes.end()) { 229 if (timestampNs - lastFlushNs->second < kMinBroadcastPeriod) { 230 return; 231 } 232 } 233 mLastBroadcastTimes[key] = timestampNs; 234 VLOG("StatsD requesting broadcast for %s", key.ToString().c_str()); 235 mSendBroadcast(key); 236 StatsdStats::getInstance().noteBroadcastSent(key); 237 } else if (totalBytes > kMaxSerializedBytes) { // Too late. We need to start clearing data. 238 // We ignore the return value so we force each metric producer to clear its contents. 239 metricsManager->onDumpReport(); 240 StatsdStats::getInstance().noteDataDropped(key); 241 VLOG("StatsD had to toss out metrics for %s", key.ToString().c_str()); 242 } 243} 244 245void StatsLogProcessor::WriteDataToDisk() { 246 mkdir(STATS_DATA_DIR, S_IRWXU); 247 for (auto& pair : mMetricsManagers) { 248 const ConfigKey& key = pair.first; 249 vector<uint8_t> data; 250 onDumpReport(key, &data); 251 // TODO: Add a guardrail to prevent accumulation of file on disk. 252 string file_name = StringPrintf("%s/%d-%s-%ld", STATS_DATA_DIR, key.GetUid(), 253 key.GetName().c_str(), time(nullptr)); 254 StorageManager::writeFile(file_name.c_str(), &data[0], data.size()); 255 } 256} 257 258} // namespace statsd 259} // namespace os 260} // namespace android 261