StatsLogProcessor.cpp revision b142cc8add29c8c97f6134d35873d23db666027c
1/* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17#define DEBUG true // STOPSHIP if true 18#include "Log.h" 19#include "statslog.h" 20 21#include <android-base/file.h> 22#include <dirent.h> 23#include "StatsLogProcessor.h" 24#include "stats_log_util.h" 25#include "android-base/stringprintf.h" 26#include "guardrail/StatsdStats.h" 27#include "metrics/CountMetricProducer.h" 28#include "external/StatsPullerManager.h" 29#include "stats_util.h" 30#include "storage/StorageManager.h" 31 32#include <log/log_event_list.h> 33#include <utils/Errors.h> 34#include <utils/SystemClock.h> 35 36using namespace android; 37using android::base::StringPrintf; 38using android::util::FIELD_COUNT_REPEATED; 39using android::util::FIELD_TYPE_BOOL; 40using android::util::FIELD_TYPE_FLOAT; 41using android::util::FIELD_TYPE_INT32; 42using android::util::FIELD_TYPE_INT64; 43using android::util::FIELD_TYPE_MESSAGE; 44using android::util::FIELD_TYPE_STRING; 45using android::util::ProtoOutputStream; 46using std::make_unique; 47using std::unique_ptr; 48using std::vector; 49 50namespace android { 51namespace os { 52namespace statsd { 53 54// for ConfigMetricsReportList 55const int FIELD_ID_CONFIG_KEY = 1; 56const int FIELD_ID_REPORTS = 2; 57// for ConfigKey 58const int FIELD_ID_UID = 1; 59const int FIELD_ID_ID = 2; 60// for ConfigMetricsReport 61// const int FIELD_ID_METRICS = 1; // written in MetricsManager.cpp 62const int FIELD_ID_UID_MAP = 2; 63const int FIELD_ID_LAST_REPORT_ELAPSED_NANOS = 3; 64const int FIELD_ID_CURRENT_REPORT_ELAPSED_NANOS = 4; 65const int FIELD_ID_LAST_REPORT_WALL_CLOCK_NANOS = 5; 66const int FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS = 6; 67 68#define NS_PER_HOUR 3600 * NS_PER_SEC 69 70#define STATS_DATA_DIR "/data/misc/stats-data" 71 72StatsLogProcessor::StatsLogProcessor(const sp<UidMap>& uidMap, 73 const sp<AlarmMonitor>& anomalyAlarmMonitor, 74 const sp<AlarmMonitor>& periodicAlarmMonitor, 75 const long timeBaseSec, 76 const std::function<void(const ConfigKey&)>& sendBroadcast) 77 : mUidMap(uidMap), 78 mAnomalyAlarmMonitor(anomalyAlarmMonitor), 79 mPeriodicAlarmMonitor(periodicAlarmMonitor), 80 mSendBroadcast(sendBroadcast), 81 mTimeBaseSec(timeBaseSec), 82 mLastLogTimestamp(0) { 83} 84 85StatsLogProcessor::~StatsLogProcessor() { 86} 87 88void StatsLogProcessor::onAnomalyAlarmFired( 89 const int64_t& timestampNs, 90 unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>> alarmSet) { 91 std::lock_guard<std::mutex> lock(mMetricsMutex); 92 for (const auto& itr : mMetricsManagers) { 93 itr.second->onAnomalyAlarmFired(timestampNs, alarmSet); 94 } 95} 96void StatsLogProcessor::onPeriodicAlarmFired( 97 const int64_t& timestampNs, 98 unordered_set<sp<const InternalAlarm>, SpHash<InternalAlarm>> alarmSet) { 99 100 std::lock_guard<std::mutex> lock(mMetricsMutex); 101 for (const auto& itr : mMetricsManagers) { 102 itr.second->onPeriodicAlarmFired(timestampNs, alarmSet); 103 } 104} 105 106void updateUid(Value* value, int hostUid) { 107 int uid = value->int_value; 108 if (uid != hostUid) { 109 value->setInt(hostUid); 110 } 111} 112 113void StatsLogProcessor::mapIsolatedUidToHostUidIfNecessaryLocked(LogEvent* event) const { 114 if (android::util::AtomsInfo::kAtomsWithAttributionChain.find(event->GetTagId()) != 115 android::util::AtomsInfo::kAtomsWithAttributionChain.end()) { 116 for (auto& value : *(event->getMutableValues())) { 117 if (value.mField.getPosAtDepth(0) > kAttributionField) { 118 break; 119 } 120 if (isAttributionUidField(value)) { 121 const int hostUid = mUidMap->getHostUidOrSelf(value.mValue.int_value); 122 updateUid(&value.mValue, hostUid); 123 } 124 } 125 } else { 126 auto it = android::util::AtomsInfo::kAtomsWithUidField.find(event->GetTagId()); 127 if (it != android::util::AtomsInfo::kAtomsWithUidField.end()) { 128 int uidField = it->second; // uidField is the field number in proto, 129 // starting from 1 130 if (uidField > 0 && (int)event->getValues().size() >= uidField && 131 (event->getValues())[uidField - 1].mValue.getType() == INT) { 132 Value& value = (*event->getMutableValues())[uidField - 1].mValue; 133 const int hostUid = mUidMap->getHostUidOrSelf(value.int_value); 134 updateUid(&value, hostUid); 135 } else { 136 ALOGE("Malformed log, uid not found. %s", event->ToString().c_str()); 137 } 138 } 139 } 140} 141 142void StatsLogProcessor::onIsolatedUidChangedEventLocked(const LogEvent& event) { 143 status_t err = NO_ERROR, err2 = NO_ERROR, err3 = NO_ERROR; 144 bool is_create = event.GetBool(3, &err); 145 auto parent_uid = int(event.GetLong(1, &err2)); 146 auto isolated_uid = int(event.GetLong(2, &err3)); 147 if (err == NO_ERROR && err2 == NO_ERROR && err3 == NO_ERROR) { 148 if (is_create) { 149 mUidMap->assignIsolatedUid(isolated_uid, parent_uid); 150 } else { 151 mUidMap->removeIsolatedUid(isolated_uid, parent_uid); 152 } 153 } else { 154 ALOGE("Failed to parse uid in the isolated uid change event."); 155 } 156} 157 158void StatsLogProcessor::OnLogEvent(LogEvent* event) { 159 std::lock_guard<std::mutex> lock(mMetricsMutex); 160 const int64_t currentTimestampNs = event->GetElapsedTimestampNs(); 161 if (currentTimestampNs < mLastLogTimestamp) { 162 return; 163 } 164 165 resetIfConfigTtlExpiredLocked(currentTimestampNs); 166 167 mLastLogTimestamp = currentTimestampNs; 168 StatsdStats::getInstance().noteAtomLogged( 169 event->GetTagId(), event->GetElapsedTimestampNs() / NS_PER_SEC); 170 171 // Hard-coded logic to update the isolated uid's in the uid-map. 172 // The field numbers need to be currently updated by hand with atoms.proto 173 if (event->GetTagId() == android::util::ISOLATED_UID_CHANGED) { 174 onIsolatedUidChangedEventLocked(*event); 175 } 176 177 if (mMetricsManagers.empty()) { 178 return; 179 } 180 181 int64_t curTimeSec = getElapsedRealtimeSec(); 182 if (curTimeSec - mLastPullerCacheClearTimeSec > StatsdStats::kPullerCacheClearIntervalSec) { 183 mStatsPullerManager.ClearPullerCacheIfNecessary(curTimeSec * NS_PER_SEC); 184 mLastPullerCacheClearTimeSec = curTimeSec; 185 } 186 187 188 if (event->GetTagId() != android::util::ISOLATED_UID_CHANGED) { 189 // Map the isolated uid to host uid if necessary. 190 mapIsolatedUidToHostUidIfNecessaryLocked(event); 191 } 192 193 // pass the event to metrics managers. 194 for (auto& pair : mMetricsManagers) { 195 pair.second->onLogEvent(*event); 196 flushIfNecessaryLocked(event->GetElapsedTimestampNs(), pair.first, *(pair.second)); 197 } 198} 199 200void StatsLogProcessor::OnConfigUpdated(const int64_t timestampNs, const ConfigKey& key, 201 const StatsdConfig& config) { 202 std::lock_guard<std::mutex> lock(mMetricsMutex); 203 OnConfigUpdatedLocked(timestampNs, key, config); 204} 205 206void StatsLogProcessor::OnConfigUpdatedLocked( 207 const int64_t timestampNs, const ConfigKey& key, const StatsdConfig& config) { 208 VLOG("Updated configuration for key %s", key.ToString().c_str()); 209 sp<MetricsManager> newMetricsManager = 210 new MetricsManager(key, config, mTimeBaseSec, (timestampNs - 1) / NS_PER_SEC + 1, mUidMap, 211 mAnomalyAlarmMonitor, mPeriodicAlarmMonitor); 212 213 if (newMetricsManager->isConfigValid()) { 214 mUidMap->OnConfigUpdated(key); 215 if (newMetricsManager->shouldAddUidMapListener()) { 216 // We have to add listener after the MetricsManager is constructed because it's 217 // not safe to create wp or sp from this pointer inside its constructor. 218 mUidMap->addListener(newMetricsManager.get()); 219 } 220 newMetricsManager->refreshTtl(timestampNs); 221 mMetricsManagers[key] = newMetricsManager; 222 VLOG("StatsdConfig valid"); 223 } else { 224 // If there is any error in the config, don't use it. 225 ALOGE("StatsdConfig NOT valid"); 226 } 227} 228 229size_t StatsLogProcessor::GetMetricsSize(const ConfigKey& key) const { 230 std::lock_guard<std::mutex> lock(mMetricsMutex); 231 auto it = mMetricsManagers.find(key); 232 if (it == mMetricsManagers.end()) { 233 ALOGW("Config source %s does not exist", key.ToString().c_str()); 234 return 0; 235 } 236 return it->second->byteSize(); 237} 238 239void StatsLogProcessor::dumpStates(FILE* out, bool verbose) { 240 std::lock_guard<std::mutex> lock(mMetricsMutex); 241 fprintf(out, "MetricsManager count: %lu\n", (unsigned long)mMetricsManagers.size()); 242 for (auto metricsManager : mMetricsManagers) { 243 metricsManager.second->dumpStates(out, verbose); 244 } 245} 246 247/* 248 * onDumpReport dumps serialized ConfigMetricsReportList into outData. 249 */ 250void StatsLogProcessor::onDumpReport(const ConfigKey& key, const int64_t dumpTimeStampNs, 251 vector<uint8_t>* outData) { 252 std::lock_guard<std::mutex> lock(mMetricsMutex); 253 254 auto it = mMetricsManagers.find(key); 255 if (it == mMetricsManagers.end()) { 256 ALOGW("Config source %s does not exist", key.ToString().c_str()); 257 return; 258 } 259 260 // This allows another broadcast to be sent within the rate-limit period if we get close to 261 // filling the buffer again soon. 262 mLastBroadcastTimes.erase(key); 263 264 ProtoOutputStream proto; 265 266 // Start of ConfigKey. 267 uint64_t configKeyToken = proto.start(FIELD_TYPE_MESSAGE | FIELD_ID_CONFIG_KEY); 268 proto.write(FIELD_TYPE_INT32 | FIELD_ID_UID, key.GetUid()); 269 proto.write(FIELD_TYPE_INT64 | FIELD_ID_ID, (long long)key.GetId()); 270 proto.end(configKeyToken); 271 // End of ConfigKey. 272 273 // Start of ConfigMetricsReport (reports). 274 uint64_t reportsToken = 275 proto.start(FIELD_TYPE_MESSAGE | FIELD_COUNT_REPEATED | FIELD_ID_REPORTS); 276 onConfigMetricsReportLocked(key, dumpTimeStampNs, &proto); 277 proto.end(reportsToken); 278 // End of ConfigMetricsReport (reports). 279 280 281 // Then, check stats-data directory to see there's any file containing 282 // ConfigMetricsReport from previous shutdowns to concatenate to reports. 283 StorageManager::appendConfigMetricsReport(key, &proto); 284 285 if (outData != nullptr) { 286 outData->clear(); 287 outData->resize(proto.size()); 288 size_t pos = 0; 289 auto iter = proto.data(); 290 while (iter.readBuffer() != NULL) { 291 size_t toRead = iter.currentToRead(); 292 std::memcpy(&((*outData)[pos]), iter.readBuffer(), toRead); 293 pos += toRead; 294 iter.rp()->move(toRead); 295 } 296 } 297 298 StatsdStats::getInstance().noteMetricsReportSent(key); 299} 300 301/* 302 * onConfigMetricsReportLocked dumps serialized ConfigMetricsReport into outData. 303 */ 304void StatsLogProcessor::onConfigMetricsReportLocked(const ConfigKey& key, 305 const int64_t dumpTimeStampNs, 306 ProtoOutputStream* proto) { 307 // We already checked whether key exists in mMetricsManagers in 308 // WriteDataToDisk. 309 auto it = mMetricsManagers.find(key); 310 int64_t lastReportTimeNs = it->second->getLastReportTimeNs(); 311 int64_t lastReportWallClockNs = it->second->getLastReportWallClockNs(); 312 313 // First, fill in ConfigMetricsReport using current data on memory, which 314 // starts from filling in StatsLogReport's. 315 it->second->onDumpReport(dumpTimeStampNs, proto); 316 317 // Fill in UidMap. 318 uint64_t uidMapToken = proto->start(FIELD_TYPE_MESSAGE | FIELD_ID_UID_MAP); 319 mUidMap->appendUidMap(key, proto); 320 proto->end(uidMapToken); 321 322 // Fill in the timestamps. 323 proto->write(FIELD_TYPE_INT64 | FIELD_ID_LAST_REPORT_ELAPSED_NANOS, 324 (long long)lastReportTimeNs); 325 proto->write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_ELAPSED_NANOS, 326 (long long)dumpTimeStampNs); 327 proto->write(FIELD_TYPE_INT64 | FIELD_ID_LAST_REPORT_WALL_CLOCK_NANOS, 328 (long long)lastReportWallClockNs); 329 proto->write(FIELD_TYPE_INT64 | FIELD_ID_CURRENT_REPORT_WALL_CLOCK_NANOS, 330 (long long)getWallClockNs()); 331 332} 333 334void StatsLogProcessor::resetIfConfigTtlExpiredLocked(const int64_t timestampNs) { 335 std::vector<ConfigKey> configKeysTtlExpired; 336 for (auto it = mMetricsManagers.begin(); it != mMetricsManagers.end(); it++) { 337 if (it->second != nullptr && !it->second->isInTtl(timestampNs)) { 338 configKeysTtlExpired.push_back(it->first); 339 } 340 } 341 342 for (const auto& key : configKeysTtlExpired) { 343 StatsdConfig config; 344 if (StorageManager::readConfigFromDisk(key, &config)) { 345 OnConfigUpdatedLocked(timestampNs, key, config); 346 StatsdStats::getInstance().noteConfigReset(key); 347 } else { 348 ALOGE("Failed to read backup config from disk for : %s", key.ToString().c_str()); 349 auto it = mMetricsManagers.find(key); 350 if (it != mMetricsManagers.end()) { 351 it->second->refreshTtl(timestampNs); 352 } 353 } 354 } 355} 356 357void StatsLogProcessor::OnConfigRemoved(const ConfigKey& key) { 358 std::lock_guard<std::mutex> lock(mMetricsMutex); 359 auto it = mMetricsManagers.find(key); 360 if (it != mMetricsManagers.end()) { 361 mMetricsManagers.erase(it); 362 mUidMap->OnConfigRemoved(key); 363 } 364 StatsdStats::getInstance().noteConfigRemoved(key); 365 366 mLastBroadcastTimes.erase(key); 367 368 if (mMetricsManagers.empty()) { 369 mStatsPullerManager.ForceClearPullerCache(); 370 } 371} 372 373void StatsLogProcessor::flushIfNecessaryLocked( 374 int64_t timestampNs, const ConfigKey& key, MetricsManager& metricsManager) { 375 auto lastCheckTime = mLastByteSizeTimes.find(key); 376 if (lastCheckTime != mLastByteSizeTimes.end()) { 377 if (timestampNs - lastCheckTime->second < StatsdStats::kMinByteSizeCheckPeriodNs) { 378 return; 379 } 380 } 381 382 // We suspect that the byteSize() computation is expensive, so we set a rate limit. 383 size_t totalBytes = metricsManager.byteSize(); 384 mLastByteSizeTimes[key] = timestampNs; 385 if (totalBytes > 386 StatsdStats::kMaxMetricsBytesPerConfig) { // Too late. We need to start clearing data. 387 metricsManager.dropData(timestampNs); 388 StatsdStats::getInstance().noteDataDropped(key); 389 VLOG("StatsD had to toss out metrics for %s", key.ToString().c_str()); 390 } else if (totalBytes > StatsdStats::kBytesPerConfigTriggerGetData) { 391 // Send broadcast so that receivers can pull data. 392 auto lastBroadcastTime = mLastBroadcastTimes.find(key); 393 if (lastBroadcastTime != mLastBroadcastTimes.end()) { 394 if (timestampNs - lastBroadcastTime->second < StatsdStats::kMinBroadcastPeriodNs) { 395 VLOG("StatsD would've sent a broadcast but the rate limit stopped us."); 396 return; 397 } 398 } 399 mLastBroadcastTimes[key] = timestampNs; 400 VLOG("StatsD requesting broadcast for %s", key.ToString().c_str()); 401 mSendBroadcast(key); 402 StatsdStats::getInstance().noteBroadcastSent(key); 403 } 404} 405 406void StatsLogProcessor::WriteDataToDisk() { 407 std::lock_guard<std::mutex> lock(mMetricsMutex); 408 for (auto& pair : mMetricsManagers) { 409 const ConfigKey& key = pair.first; 410 ProtoOutputStream proto; 411 onConfigMetricsReportLocked(key, getElapsedRealtimeNs(), &proto); 412 string file_name = StringPrintf("%s/%ld_%d_%lld", STATS_DATA_DIR, 413 (long)getWallClockSec(), key.GetUid(), (long long)key.GetId()); 414 android::base::unique_fd fd(open(file_name.c_str(), 415 O_WRONLY | O_CREAT | O_CLOEXEC, S_IRUSR | S_IWUSR)); 416 if (fd == -1) { 417 VLOG("Attempt to write %s but failed", file_name.c_str()); 418 return; 419 } 420 proto.flush(fd.get()); 421 } 422} 423 424} // namespace statsd 425} // namespace os 426} // namespace android 427