StatsLogProcessor.cpp revision f384b900498b552c776931c8873e4bd03af36fe5
1/* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#define DEBUG true // STOPSHIP if true 18#include "Log.h" 19#include "statslog.h" 20 21#include <android-base/file.h> 22#include <dirent.h> 23#include "StatsLogProcessor.h" 24#include "stats_log_util.h" 25#include "android-base/stringprintf.h" 26#include "guardrail/StatsdStats.h" 27#include "metrics/CountMetricProducer.h" 28#include "external/StatsPullerManager.h" 29#include "stats_util.h" 30#include "storage/StorageManager.h" 31 32#include <log/log_event_list.h> 33#include <utils/Errors.h> 34#include <utils/SystemClock.h> 35 36using namespace android; 37using android::base::StringPrintf; 38using android::util::FIELD_COUNT_REPEATED; 39using android::util::FIELD_TYPE_BOOL; 40using android::util::FIELD_TYPE_FLOAT; 41using android::util::FIELD_TYPE_INT32; 42using android::util::FIELD_TYPE_INT64; 43using android::util::FIELD_TYPE_MESSAGE; 44using android::util::FIELD_TYPE_STRING; 45using android::util::ProtoOutputStream; 46using std::make_unique; 47using std::unique_ptr; 48using std::vector; 49 50namespace android { 51namespace os { 52namespace statsd { 53 54// for ConfigMetricsReportList 55const int FIELD_ID_CONFIG_KEY = 1; 56const int FIELD_ID_REPORTS = 2; 57// for ConfigKey 58const int FIELD_ID_UID = 1; 59const int FIELD_ID_ID = 2; 60// for ConfigMetricsReport 61// const int FIELD_ID_METRICS = 1; // written in MetricsManager.cpp 62const int FIELD_ID_UID_MAP = 2; 63const int FIELD_ID_LAST_REPORT_ELAPSED_NANOS = 3; 64const int FIELD_ID_CURRENT_REPORT_ELAPSED_NANOS = 4; 65const int FIELD_ID_LAST_REPORT_WALL_CLOCK_NANOS = 5; 66const int FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS = 6; 67 68 69#define STATS_DATA_DIR "/data/misc/stats-data" 70 71StatsLogProcessor::StatsLogProcessor(const sp<UidMap>& uidMap, 72 const sp<AlarmMonitor>& anomalyAlarmMonitor, 73 const sp<AlarmMonitor>& periodicAlarmMonitor, 74 const long timeBaseSec, 75 const std::function<void(const ConfigKey&)>& sendBroadcast) 76 : mUidMap(uidMap), 77 mAnomalyAlarmMonitor(anomalyAlarmMonitor), 78 mPeriodicAlarmMonitor(periodicAlarmMonitor), 79 mSendBroadcast(sendBroadcast), 80 mTimeBaseSec(timeBaseSec), 81 mLastLogTimestamp(0) { 82 StatsPullerManager statsPullerManager; 83 statsPullerManager.SetTimeBaseSec(mTimeBaseSec); 84} 85 86StatsLogProcessor::~StatsLogProcessor() { 87} 88 89void StatsLogProcessor::onAnomalyAlarmFired( 90 const uint64_t timestampNs, 91 unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>> alarmSet) { 92 std::lock_guard<std::mutex> lock(mMetricsMutex); 93 for (const auto& itr : mMetricsManagers) { 94 itr.second->onAnomalyAlarmFired(timestampNs, alarmSet); 95 } 96} 97void StatsLogProcessor::onPeriodicAlarmFired( 98 const uint64_t timestampNs, 99 unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>> alarmSet) { 100 101 std::lock_guard<std::mutex> lock(mMetricsMutex); 102 for (const auto& itr : mMetricsManagers) { 103 itr.second->onPeriodicAlarmFired(timestampNs, alarmSet); 104 } 105} 106 107void updateUid(Value* value, int hostUid) { 108 int uid = value->int_value; 109 if (uid != hostUid) { 110 value->setInt(hostUid); 111 } 112} 113 114void StatsLogProcessor::mapIsolatedUidToHostUidIfNecessaryLocked(LogEvent* event) const { 115 if (android::util::kAtomsWithAttributionChain.find(event->GetTagId()) != 116 android::util::kAtomsWithAttributionChain.end()) { 117 for (auto& value : *(event->getMutableValues())) { 118 if (value.mField.getPosAtDepth(0) > kAttributionField) { 119 break; 120 } 121 if (isAttributionUidField(value)) { 122 const int hostUid = mUidMap->getHostUidOrSelf(value.mValue.int_value); 123 updateUid(&value.mValue, hostUid); 124 } 125 } 126 } else if (android::util::kAtomsWithUidField.find(event->GetTagId()) != 127 android::util::kAtomsWithUidField.end() && 128 event->getValues().size() > 0 && (event->getValues())[0].mValue.getType() == INT) { 129 Value& value = (*event->getMutableValues())[0].mValue; 130 const int hostUid = mUidMap->getHostUidOrSelf(value.int_value); 131 updateUid(&value, hostUid); 132 } 133} 134 135void StatsLogProcessor::onIsolatedUidChangedEventLocked(const LogEvent& event) { 136 status_t err = NO_ERROR, err2 = NO_ERROR, err3 = NO_ERROR; 137 bool is_create = event.GetBool(3, &err); 138 auto parent_uid = int(event.GetLong(1, &err2)); 139 auto isolated_uid = int(event.GetLong(2, &err3)); 140 if (err == NO_ERROR && err2 == NO_ERROR && err3 == NO_ERROR) { 141 if (is_create) { 142 mUidMap->assignIsolatedUid(isolated_uid, parent_uid); 143 } else { 144 mUidMap->removeIsolatedUid(isolated_uid, parent_uid); 145 } 146 } else { 147 ALOGE("Failed to parse uid in the isolated uid change event."); 148 } 149} 150 151void StatsLogProcessor::OnLogEvent(LogEvent* event) { 152 std::lock_guard<std::mutex> lock(mMetricsMutex); 153 if (event->GetElapsedTimestampNs() < mLastLogTimestamp) { 154 return; 155 } 156 mLastLogTimestamp = event->GetElapsedTimestampNs(); 157 StatsdStats::getInstance().noteAtomLogged( 158 event->GetTagId(), event->GetElapsedTimestampNs() / NS_PER_SEC); 159 160 // Hard-coded logic to update the isolated uid's in the uid-map. 161 // The field numbers need to be currently updated by hand with atoms.proto 162 if (event->GetTagId() == android::util::ISOLATED_UID_CHANGED) { 163 onIsolatedUidChangedEventLocked(*event); 164 } 165 166 if (mMetricsManagers.empty()) { 167 return; 168 } 169 170 uint64_t curTimeSec = getElapsedRealtimeSec(); 171 if (curTimeSec - mLastPullerCacheClearTimeSec > StatsdStats::kPullerCacheClearIntervalSec) { 172 mStatsPullerManager.ClearPullerCacheIfNecessary(curTimeSec); 173 mLastPullerCacheClearTimeSec = curTimeSec; 174 } 175 176 if (event->GetTagId() != android::util::ISOLATED_UID_CHANGED) { 177 // Map the isolated uid to host uid if necessary. 178 mapIsolatedUidToHostUidIfNecessaryLocked(event); 179 } 180 181 // pass the event to metrics managers. 182 for (auto& pair : mMetricsManagers) { 183 pair.second->onLogEvent(*event); 184 flushIfNecessaryLocked(event->GetElapsedTimestampNs(), pair.first, *(pair.second)); 185 } 186} 187 188void StatsLogProcessor::OnConfigUpdated(const ConfigKey& key, const StatsdConfig& config) { 189 std::lock_guard<std::mutex> lock(mMetricsMutex); 190 VLOG("Updated configuration for key %s", key.ToString().c_str()); 191 sp<MetricsManager> newMetricsManager = 192 new MetricsManager(key, config, mTimeBaseSec, mUidMap, 193 mAnomalyAlarmMonitor, mPeriodicAlarmMonitor); 194 auto it = mMetricsManagers.find(key); 195 if (it == mMetricsManagers.end() && mMetricsManagers.size() > StatsdStats::kMaxConfigCount) { 196 ALOGE("Can't accept more configs!"); 197 return; 198 } 199 200 if (newMetricsManager->isConfigValid()) { 201 mUidMap->OnConfigUpdated(key); 202 if (newMetricsManager->shouldAddUidMapListener()) { 203 // We have to add listener after the MetricsManager is constructed because it's 204 // not safe to create wp or sp from this pointer inside its constructor. 205 mUidMap->addListener(newMetricsManager.get()); 206 } 207 mMetricsManagers[key] = newMetricsManager; 208 // Why doesn't this work? mMetricsManagers.insert({key, std::move(newMetricsManager)}); 209 VLOG("StatsdConfig valid"); 210 } else { 211 // If there is any error in the config, don't use it. 212 ALOGE("StatsdConfig NOT valid"); 213 } 214} 215 216size_t StatsLogProcessor::GetMetricsSize(const ConfigKey& key) const { 217 std::lock_guard<std::mutex> lock(mMetricsMutex); 218 auto it = mMetricsManagers.find(key); 219 if (it == mMetricsManagers.end()) { 220 ALOGW("Config source %s does not exist", key.ToString().c_str()); 221 return 0; 222 } 223 return it->second->byteSize(); 224} 225 226void StatsLogProcessor::dumpStates(FILE* out, bool verbose) { 227 std::lock_guard<std::mutex> lock(mMetricsMutex); 228 fprintf(out, "MetricsManager count: %lu\n", (unsigned long)mMetricsManagers.size()); 229 for (auto metricsManager : mMetricsManagers) { 230 metricsManager.second->dumpStates(out, verbose); 231 } 232} 233 234void StatsLogProcessor::onDumpReport(const ConfigKey& key, const uint64_t dumpTimeStampNs, 235 vector<uint8_t>* outData) { 236 std::lock_guard<std::mutex> lock(mMetricsMutex); 237 onDumpReportLocked(key, dumpTimeStampNs, outData); 238} 239 240void StatsLogProcessor::onDumpReportLocked(const ConfigKey& key, const uint64_t dumpTimeStampNs, 241 vector<uint8_t>* outData) { 242 auto it = mMetricsManagers.find(key); 243 if (it == mMetricsManagers.end()) { 244 ALOGW("Config source %s does not exist", key.ToString().c_str()); 245 return; 246 } 247 248 // This allows another broadcast to be sent within the rate-limit period if we get close to 249 // filling the buffer again soon. 250 mLastBroadcastTimes.erase(key); 251 252 ProtoOutputStream proto; 253 254 // Start of ConfigKey. 255 uint64_t configKeyToken = proto.start(FIELD_TYPE_MESSAGE | FIELD_ID_CONFIG_KEY); 256 proto.write(FIELD_TYPE_INT32 | FIELD_ID_UID, key.GetUid()); 257 proto.write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)key.GetId()); 258 proto.end(configKeyToken); 259 // End of ConfigKey. 260 261 // Start of ConfigMetricsReport (reports). 262 uint64_t reportsToken = 263 proto.start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS); 264 265 int64_t lastReportTimeNs = it->second->getLastReportTimeNs(); 266 int64_t lastReportWallClockNs = it->second->getLastReportWallClockNs(); 267 268 // First, fill in ConfigMetricsReport using current data on memory, which 269 // starts from filling in StatsLogReport's. 270 it->second->onDumpReport(dumpTimeStampNs, &proto); 271 272 // Fill in UidMap. 273 vector<uint8_t> uidMap; 274 mUidMap->getOutput(key, &uidMap); 275 proto.write(FIELD_TYPE_MESSAGE | FIELD_ID_UID_MAP, uidMap.data()); 276 277 // Fill in the timestamps. 278 proto.write(FIELD_TYPE_INT64 | FIELD_ID_LAST_REPORT_ELAPSED_NANOS, 279 (long long)lastReportTimeNs); 280 proto.write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_ELAPSED_NANOS, 281 (long long)dumpTimeStampNs); 282 proto.write(FIELD_TYPE_INT64 | FIELD_ID_LAST_REPORT_WALL_CLOCK_NANOS, 283 (long long)lastReportWallClockNs); 284 proto.write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS, 285 (long long)getWallClockNs()); 286 287 // End of ConfigMetricsReport (reports). 288 proto.end(reportsToken); 289 290 // Then, check stats-data directory to see there's any file containing 291 // ConfigMetricsReport from previous shutdowns to concatenate to reports. 292 StorageManager::appendConfigMetricsReport(proto); 293 294 if (outData != nullptr) { 295 outData->clear(); 296 outData->resize(proto.size()); 297 size_t pos = 0; 298 auto iter = proto.data(); 299 while (iter.readBuffer() != NULL) { 300 size_t toRead = iter.currentToRead(); 301 std::memcpy(&((*outData)[pos]), iter.readBuffer(), toRead); 302 pos += toRead; 303 iter.rp()->move(toRead); 304 } 305 } 306 307 StatsdStats::getInstance().noteMetricsReportSent(key); 308} 309 310void StatsLogProcessor::OnConfigRemoved(const ConfigKey& key) { 311 std::lock_guard<std::mutex> lock(mMetricsMutex); 312 auto it = mMetricsManagers.find(key); 313 if (it != mMetricsManagers.end()) { 314 mMetricsManagers.erase(it); 315 mUidMap->OnConfigRemoved(key); 316 } 317 StatsdStats::getInstance().noteConfigRemoved(key); 318 319 mLastBroadcastTimes.erase(key); 320 321 if (mMetricsManagers.empty()) { 322 mStatsPullerManager.ForceClearPullerCache(); 323 } 324} 325 326void StatsLogProcessor::flushIfNecessaryLocked( 327 uint64_t timestampNs, const ConfigKey& key, MetricsManager& metricsManager) { 328 auto lastCheckTime = mLastByteSizeTimes.find(key); 329 if (lastCheckTime != mLastByteSizeTimes.end()) { 330 if (timestampNs - lastCheckTime->second < StatsdStats::kMinByteSizeCheckPeriodNs) { 331 return; 332 } 333 } 334 335 // We suspect that the byteSize() computation is expensive, so we set a rate limit. 336 size_t totalBytes = metricsManager.byteSize(); 337 mLastByteSizeTimes[key] = timestampNs; 338 if (totalBytes > 339 StatsdStats::kMaxMetricsBytesPerConfig) { // Too late. We need to start clearing data. 340 metricsManager.dropData(timestampNs); 341 StatsdStats::getInstance().noteDataDropped(key); 342 VLOG("StatsD had to toss out metrics for %s", key.ToString().c_str()); 343 } else if (totalBytes > .9 * StatsdStats::kMaxMetricsBytesPerConfig) { 344 // Send broadcast so that receivers can pull data. 345 auto lastBroadcastTime = mLastBroadcastTimes.find(key); 346 if (lastBroadcastTime != mLastBroadcastTimes.end()) { 347 if (timestampNs - lastBroadcastTime->second < StatsdStats::kMinBroadcastPeriodNs) { 348 VLOG("StatsD would've sent a broadcast but the rate limit stopped us."); 349 return; 350 } 351 } 352 mLastBroadcastTimes[key] = timestampNs; 353 VLOG("StatsD requesting broadcast for %s", key.ToString().c_str()); 354 mSendBroadcast(key); 355 StatsdStats::getInstance().noteBroadcastSent(key); 356 } 357} 358 359void StatsLogProcessor::WriteDataToDisk() { 360 std::lock_guard<std::mutex> lock(mMetricsMutex); 361 for (auto& pair : mMetricsManagers) { 362 const ConfigKey& key = pair.first; 363 vector<uint8_t> data; 364 onDumpReportLocked(key, getElapsedRealtimeNs(), &data); 365 string file_name = StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR, 366 (long)getWallClockSec(), key.GetUid(), (long long)key.GetId()); 367 StorageManager::writeFile(file_name.c_str(), &data[0], data.size()); 368 } 369} 370 371} // namespace statsd 372} // namespace os 373} // namespace android 374