field_trial.h revision 3f50c38dc070f4bb515c1b64450dae14f316474e
1// Copyright (c) 2011 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5// FieldTrial is a class for handling details of statistical experiments 6// performed by actual users in the field (i.e., in a shipped or beta product). 7// All code is called exclusively on the UI thread currently. 8// 9// The simplest example is an experiment to see whether one of two options 10// produces "better" results across our user population. In that scenario, UMA 11// data is uploaded to aggregate the test results, and this FieldTrial class 12// manages the state of each such experiment (state == which option was 13// pseudo-randomly selected). 14// 15// States are typically generated randomly, either based on a one time 16// randomization (generated randomly once, and then persistently reused in the 17// client during each future run of the program), or by a startup randomization 18// (generated each time the application starts up, but held constant during the 19// duration of the process), or by continuous randomization across a run (where 20// the state can be recalculated again and again, many times during a process). 21// Only startup randomization is implemented thus far. 22 23//------------------------------------------------------------------------------ 24// Example: Suppose we have an experiment involving memory, such as determining 25// the impact of some pruning algorithm. 26// We assume that we already have a histogram of memory usage, such as: 27 28// HISTOGRAM_COUNTS("Memory.RendererTotal", count); 29 30// Somewhere in main thread initialization code, we'd probably define an 31// instance of a FieldTrial, with code such as: 32 33// // Note, FieldTrials are reference counted, and persist automagically until 34// // process teardown, courtesy of their automatic registration in 35// // FieldTrialList. 36// scoped_refptr<FieldTrial> trial = new FieldTrial("MemoryExperiment", 1000); 37// int group1 = trial->AppendGroup("high_mem", 20); // 2% in high_mem group. 38// int group2 = trial->AppendGroup("low_mem", 20); // 2% in low_mem group. 39// // Take action depending of which group we randomly land in. 40// if (trial->group() == group1) 41// SetPruningAlgorithm(kType1); // Sample setting of browser state. 42// else if (trial->group() == group2) 43// SetPruningAlgorithm(kType2); // Sample alternate setting. 44 45// We then modify any histograms we wish to correlate with our experiment to 46// have slighly different names, depending on what group the trial instance 47// happened (randomly) to be assigned to: 48 49// HISTOGRAM_COUNTS(FieldTrial::MakeName("Memory.RendererTotal", 50// "MemoryExperiment").data(), count); 51 52// The above code will create 3 distinct histograms, with each run of the 53// application being assigned to of of the three groups, and for each group, the 54// correspondingly named histogram will be populated: 55 56// Memory.RendererTotal // 96% of users still fill this histogram. 57// Memory.RendererTotal_high_mem // 2% of users will fill this histogram. 58// Memory.RendererTotal_low_mem // 2% of users will fill this histogram. 59 60//------------------------------------------------------------------------------ 61 62#ifndef BASE_METRICS_FIELD_TRIAL_H_ 63#define BASE_METRICS_FIELD_TRIAL_H_ 64#pragma once 65 66#include <map> 67#include <string> 68 69#include "base/lock.h" 70#include "base/ref_counted.h" 71#include "base/time.h" 72 73namespace base { 74 75class FieldTrial : public RefCounted<FieldTrial> { 76 public: 77 typedef int Probability; // Probability type for being selected in a trial. 78 79 // A return value to indicate that a given instance has not yet had a group 80 // assignment (and hence is not yet participating in the trial). 81 static const int kNotParticipating; 82 83 // Provide an easy way to assign all remaining probability to a group. Note 84 // that this will force an instance to participate, and make it illegal to 85 // attempt to probabalistically add any other groups to the trial. When doing 86 // A/B tests with timings, it is often best to define all groups, so that 87 // histograms will get unique names via the MakeName() methods. 88 static const Probability kAllRemainingProbability; 89 90 // The name is used to register the instance with the FieldTrialList class, 91 // and can be used to find the trial (only one trial can be present for each 92 // name). 93 // Group probabilities that are later supplied must sum to less than or equal 94 // to the total_probability. 95 FieldTrial(const std::string& name, Probability total_probability); 96 97 // Establish the name and probability of the next group in this trial. 98 // Sometimes, based on construction randomization, this call may causes the 99 // provided group to be *THE* group selected for use in this instance. 100 int AppendGroup(const std::string& name, Probability group_probability); 101 102 // Return the name of the FieldTrial (excluding the group name). 103 std::string name() const { return name_; } 104 105 // Return the randomly selected group number that was assigned. 106 // Return kNotParticipating if the instance is not participating in the 107 // experiment. 108 int group() const { return group_; } 109 110 // If the field trial is not in an experiment, this returns the empty string. 111 // if the group's name is empty, a name of "_" concatenated with the group 112 // number is used as the group name. 113 std::string group_name() const { return group_name_; } 114 115 // Helper function for the most common use: as an argument to specifiy the 116 // name of a HISTOGRAM. Use the original histogram name as the name_prefix. 117 static std::string MakeName(const std::string& name_prefix, 118 const std::string& trial_name); 119 120 // Enable benchmarking sets field trials to a common setting. 121 static void EnableBenchmarking(); 122 123 private: 124 friend class RefCounted<FieldTrial>; 125 126 virtual ~FieldTrial(); 127 128 // The name of the field trial, as can be found via the FieldTrialList. 129 // This is empty of the trial is not in the experiment. 130 const std::string name_; 131 132 // The maximum sum of all probabilities supplied, which corresponds to 100%. 133 // This is the scaling factor used to adjust supplied probabilities. 134 Probability divisor_; 135 136 // The randomly selected probability that is used to select a group (or have 137 // the instance not participate). It is the product of divisor_ and a random 138 // number between [0, 1). 139 Probability random_; 140 141 // Sum of the probabilities of all appended groups. 142 Probability accumulated_group_probability_; 143 144 int next_group_number_; 145 146 // The pseudo-randomly assigned group number. 147 // This is kNotParticipating if no group has been assigned. 148 int group_; 149 150 // A textual name for the randomly selected group, including the Trial name. 151 // If this Trial is not a member of an group, this string is empty. 152 std::string group_name_; 153 154 // When benchmarking is enabled, field trials all revert to the 'default' 155 // bucket. 156 static bool enable_benchmarking_; 157 158 DISALLOW_COPY_AND_ASSIGN(FieldTrial); 159}; 160 161//------------------------------------------------------------------------------ 162// Class with a list of all active field trials. A trial is active if it has 163// been registered, which includes evaluating its state based on its probaility. 164// Only one instance of this class exists. 165class FieldTrialList { 166 public: 167 // Define a separator charactor to use when creating a persistent form of an 168 // instance. This is intended for use as a command line argument, passed to a 169 // second process to mimic our state (i.e., provide the same group name). 170 static const char kPersistentStringSeparator; // Currently a slash. 171 172 // This singleton holds the global list of registered FieldTrials. 173 FieldTrialList(); 174 // Destructor Release()'s references to all registered FieldTrial instances. 175 ~FieldTrialList(); 176 177 // Register() stores a pointer to the given trial in a global map. 178 // This method also AddRef's the indicated trial. 179 static void Register(FieldTrial* trial); 180 181 // The Find() method can be used to test to see if a named Trial was already 182 // registered, or to retrieve a pointer to it from the global map. 183 static FieldTrial* Find(const std::string& name); 184 185 static int FindValue(const std::string& name); 186 187 static std::string FindFullName(const std::string& name); 188 189 // Create a persistent representation of all FieldTrial instances for 190 // resurrection in another process. This allows randomization to be done in 191 // one process, and secondary processes can by synchronized on the result. 192 // The resulting string contains only the names, the trial name, and a "/" 193 // separator. 194 static void StatesToString(std::string* output); 195 196 // Use a previously generated state string (re: StatesToString()) augment the 197 // current list of field tests to include the supplied tests, and using a 100% 198 // probability for each test, force them to have the same group string. This 199 // is commonly used in a sub-process, to carry randomly selected state in a 200 // parent process into this sub-process. 201 // Currently only the group_name_ and name_ are restored. 202 static bool StringAugmentsState(const std::string& prior_state); 203 204 // The time of construction of the global map is recorded in a static variable 205 // and is commonly used by experiments to identify the time since the start 206 // of the application. In some experiments it may be useful to discount 207 // data that is gathered before the application has reached sufficient 208 // stability (example: most DLL have loaded, etc.) 209 static TimeTicks application_start_time() { 210 if (global_) 211 return global_->application_start_time_; 212 // For testing purposes only, or when we don't yet have a start time. 213 return TimeTicks::Now(); 214 } 215 216 // Return the number of active field trials. 217 static size_t GetFieldTrialCount(); 218 219 private: 220 // A map from FieldTrial names to the actual instances. 221 typedef std::map<std::string, FieldTrial*> RegistrationList; 222 223 // Helper function should be called only while holding lock_. 224 FieldTrial* PreLockedFind(const std::string& name); 225 226 static FieldTrialList* global_; // The singleton of this class. 227 228 // This will tell us if there is an attempt to register a field trial without 229 // creating the FieldTrialList. This is not an error, unless a FieldTrialList 230 // is created after that. 231 static bool register_without_global_; 232 233 // A helper value made availabel to users, that shows when the FieldTrialList 234 // was initialized. Note that this is a singleton instance, and hence is a 235 // good approximation to the start of the process. 236 TimeTicks application_start_time_; 237 238 // Lock for access to registered_. 239 Lock lock_; 240 RegistrationList registered_; 241 242 DISALLOW_COPY_AND_ASSIGN(FieldTrialList); 243}; 244 245} // namespace base 246 247#endif // BASE_METRICS_FIELD_TRIAL_H_ 248 249