field_trial.h revision d3868032626d59662ff73b372b5d584c1d144c53
1// Copyright (c) 2012 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5// FieldTrial is a class for handling details of statistical experiments 6// performed by actual users in the field (i.e., in a shipped or beta product). 7// All code is called exclusively on the UI thread currently. 8// 9// The simplest example is an experiment to see whether one of two options 10// produces "better" results across our user population. In that scenario, UMA 11// data is uploaded to aggregate the test results, and this FieldTrial class 12// manages the state of each such experiment (state == which option was 13// pseudo-randomly selected). 14// 15// States are typically generated randomly, either based on a one time 16// randomization (which will yield the same results, in terms of selecting 17// the client for a field trial or not, for every run of the program on a 18// given machine), or by a session randomization (generated each time the 19// application starts up, but held constant during the duration of the 20// process). 21 22//------------------------------------------------------------------------------ 23// Example: Suppose we have an experiment involving memory, such as determining 24// the impact of some pruning algorithm. 25// We assume that we already have a histogram of memory usage, such as: 26 27// HISTOGRAM_COUNTS("Memory.RendererTotal", count); 28 29// Somewhere in main thread initialization code, we'd probably define an 30// instance of a FieldTrial, with code such as: 31 32// // FieldTrials are reference counted, and persist automagically until 33// // process teardown, courtesy of their automatic registration in 34// // FieldTrialList. 35// // Note: This field trial will run in Chrome instances compiled through 36// // 8 July, 2015, and after that all instances will be in "StandardMem". 37// scoped_refptr<base::FieldTrial> trial( 38// base::FieldTrialList::FactoryGetFieldTrial( 39// "MemoryExperiment", 1000, "StandardMem", 2015, 7, 8, 40// base::FieldTrial::ONE_TIME_RANDOMIZED, NULL)); 41// 42// const int high_mem_group = 43// trial->AppendGroup("HighMem", 20); // 2% in HighMem group. 44// const int low_mem_group = 45// trial->AppendGroup("LowMem", 20); // 2% in LowMem group. 46// // Take action depending of which group we randomly land in. 47// if (trial->group() == high_mem_group) 48// SetPruningAlgorithm(kType1); // Sample setting of browser state. 49// else if (trial->group() == low_mem_group) 50// SetPruningAlgorithm(kType2); // Sample alternate setting. 51 52// We then, in addition to our original histogram, output histograms which have 53// slightly different names depending on what group the trial instance happened 54// to randomly be assigned: 55 56// HISTOGRAM_COUNTS("Memory.RendererTotal", count); // The original histogram. 57// static const bool memory_renderer_total_trial_exists = 58// FieldTrialList::TrialExists("MemoryExperiment"); 59// if (memory_renderer_total_trial_exists) { 60// HISTOGRAM_COUNTS(FieldTrial::MakeName("Memory.RendererTotal", 61// "MemoryExperiment"), count); 62// } 63 64// The above code will create four distinct histograms, with each run of the 65// application being assigned to of of the three groups, and for each group, the 66// correspondingly named histogram will be populated: 67 68// Memory.RendererTotal // 100% of users still fill this histogram. 69// Memory.RendererTotal_HighMem // 2% of users will fill this histogram. 70// Memory.RendererTotal_LowMem // 2% of users will fill this histogram. 71// Memory.RendererTotal_StandardMem // 96% of users will fill this histogram. 72 73//------------------------------------------------------------------------------ 74 75#ifndef BASE_METRICS_FIELD_TRIAL_H_ 76#define BASE_METRICS_FIELD_TRIAL_H_ 77 78#include <map> 79#include <string> 80#include <vector> 81 82#include "base/base_export.h" 83#include "base/gtest_prod_util.h" 84#include "base/memory/ref_counted.h" 85#include "base/observer_list_threadsafe.h" 86#include "base/synchronization/lock.h" 87#include "base/time/time.h" 88 89namespace base { 90 91class FieldTrialList; 92 93class BASE_EXPORT FieldTrial : public RefCounted<FieldTrial> { 94 public: 95 typedef int Probability; // Probability type for being selected in a trial. 96 97 // Specifies the persistence of the field trial group choice. 98 enum RandomizationType { 99 // One time randomized trials will persist the group choice between 100 // restarts, which is recommended for most trials, especially those that 101 // change user visible behavior. 102 ONE_TIME_RANDOMIZED, 103 // Session randomized trials will roll the dice to select a group on every 104 // process restart. 105 SESSION_RANDOMIZED, 106 }; 107 108 // EntropyProvider is an interface for providing entropy for one-time 109 // randomized (persistent) field trials. 110 class BASE_EXPORT EntropyProvider { 111 public: 112 virtual ~EntropyProvider(); 113 114 // Returns a double in the range of [0, 1) to be used for the dice roll for 115 // the specified field trial. If |randomization_seed| is not 0, it will be 116 // used in preference to |trial_name| for generating the entropy by entropy 117 // providers that support it. A given instance should always return the same 118 // value given the same input |trial_name| and |randomization_seed| values. 119 virtual double GetEntropyForTrial(const std::string& trial_name, 120 uint32 randomization_seed) const = 0; 121 }; 122 123 // A pair representing a Field Trial and its selected group. 124 struct ActiveGroup { 125 std::string trial_name; 126 std::string group_name; 127 }; 128 129 typedef std::vector<ActiveGroup> ActiveGroups; 130 131 // A return value to indicate that a given instance has not yet had a group 132 // assignment (and hence is not yet participating in the trial). 133 static const int kNotFinalized; 134 135 // Disables this trial, meaning it always determines the default group 136 // has been selected. May be called immediately after construction, or 137 // at any time after initialization (should not be interleaved with 138 // AppendGroup calls). Once disabled, there is no way to re-enable a 139 // trial. 140 // TODO(mad): http://code.google.com/p/chromium/issues/detail?id=121446 141 // This doesn't properly reset to Default when a group was forced. 142 void Disable(); 143 144 // Establish the name and probability of the next group in this trial. 145 // Sometimes, based on construction randomization, this call may cause the 146 // provided group to be *THE* group selected for use in this instance. 147 // The return value is the group number of the new group. 148 int AppendGroup(const std::string& name, Probability group_probability); 149 150 // Return the name of the FieldTrial (excluding the group name). 151 const std::string& trial_name() const { return trial_name_; } 152 153 // Return the randomly selected group number that was assigned, and notify 154 // any/all observers that this finalized group number has presumably been used 155 // (queried), and will never change. Note that this will force an instance to 156 // participate, and make it illegal to attempt to probabilistically add any 157 // other groups to the trial. 158 int group(); 159 160 // If the group's name is empty, a string version containing the group number 161 // is used as the group name. This causes a winner to be chosen if none was. 162 const std::string& group_name(); 163 164 // Helper function for the most common use: as an argument to specify the 165 // name of a HISTOGRAM. Use the original histogram name as the name_prefix. 166 static std::string MakeName(const std::string& name_prefix, 167 const std::string& trial_name); 168 169 // Enable benchmarking sets field trials to a common setting. 170 static void EnableBenchmarking(); 171 172 // Set the field trial as forced, meaning that it was setup earlier than 173 // the hard coded registration of the field trial to override it. 174 // This allows the code that was hard coded to register the field trial to 175 // still succeed even though the field trial has already been registered. 176 // This must be called after appending all the groups, since we will make 177 // the group choice here. Note that this is a NOOP for already forced trials. 178 // And, as the rest of the FieldTrial code, this is not thread safe and must 179 // be done from the UI thread. 180 void SetForced(); 181 182 private: 183 // Allow tests to access our innards for testing purposes. 184 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, Registration); 185 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, AbsoluteProbabilities); 186 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, RemainingProbability); 187 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, FiftyFiftyProbability); 188 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, MiddleProbabilities); 189 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, OneWinner); 190 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, DisableProbability); 191 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, ActiveGroups); 192 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, ActiveGroupsNotFinalized); 193 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, Save); 194 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, DuplicateRestore); 195 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, MakeName); 196 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, HashClientId); 197 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, HashClientIdIsUniform); 198 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, NameGroupIds); 199 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, SetForcedTurnFeatureOff); 200 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, SetForcedTurnFeatureOn); 201 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, SetForcedChangeDefault_Default); 202 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, SetForcedChangeDefault_NonDefault); 203 204 friend class base::FieldTrialList; 205 206 friend class RefCounted<FieldTrial>; 207 208 // This is the group number of the 'default' group when a choice wasn't forced 209 // by a call to FieldTrialList::CreateFieldTrial. It is kept private so that 210 // consumers don't use it by mistake in cases where the group was forced. 211 static const int kDefaultGroupNumber; 212 213 // Creates a field trial with the specified parameters. Group assignment will 214 // be done based on |entropy_value|, which must have a range of [0, 1). 215 FieldTrial(const std::string& name, 216 Probability total_probability, 217 const std::string& default_group_name, 218 double entropy_value); 219 virtual ~FieldTrial(); 220 221 // Return the default group name of the FieldTrial. 222 std::string default_group_name() const { return default_group_name_; } 223 224 // Sets the chosen group name and number. 225 void SetGroupChoice(const std::string& group_name, int number); 226 227 // Ensures that a group is chosen, if it hasn't yet been. The field trial 228 // might yet be disabled, so this call will *not* notify observers of the 229 // status. 230 void FinalizeGroupChoice(); 231 232 // Returns the trial name and selected group name for this field trial via 233 // the output parameter |active_group|, but only if the group has already 234 // been chosen and has been externally observed via |group()| and the trial 235 // has not been disabled. In that case, true is returned and |active_group| 236 // is filled in; otherwise, the result is false and |active_group| is left 237 // untouched. 238 bool GetActiveGroup(ActiveGroup* active_group) const; 239 240 // Returns the group_name. A winner need not have been chosen. 241 std::string group_name_internal() const { return group_name_; } 242 243 // The name of the field trial, as can be found via the FieldTrialList. 244 const std::string trial_name_; 245 246 // The maximum sum of all probabilities supplied, which corresponds to 100%. 247 // This is the scaling factor used to adjust supplied probabilities. 248 const Probability divisor_; 249 250 // The name of the default group. 251 const std::string default_group_name_; 252 253 // The randomly selected probability that is used to select a group (or have 254 // the instance not participate). It is the product of divisor_ and a random 255 // number between [0, 1). 256 Probability random_; 257 258 // Sum of the probabilities of all appended groups. 259 Probability accumulated_group_probability_; 260 261 int next_group_number_; 262 263 // The pseudo-randomly assigned group number. 264 // This is kNotFinalized if no group has been assigned. 265 int group_; 266 267 // A textual name for the randomly selected group. Valid after |group()| 268 // has been called. 269 std::string group_name_; 270 271 // When enable_field_trial_ is false, field trial reverts to the 'default' 272 // group. 273 bool enable_field_trial_; 274 275 // When forced_ is true, we return the chosen group from AppendGroup when 276 // appropriate. 277 bool forced_; 278 279 // Specifies whether the group choice has been reported to observers. 280 bool group_reported_; 281 282 // When benchmarking is enabled, field trials all revert to the 'default' 283 // group. 284 static bool enable_benchmarking_; 285 286 DISALLOW_COPY_AND_ASSIGN(FieldTrial); 287}; 288 289//------------------------------------------------------------------------------ 290// Class with a list of all active field trials. A trial is active if it has 291// been registered, which includes evaluating its state based on its probaility. 292// Only one instance of this class exists. 293class BASE_EXPORT FieldTrialList { 294 public: 295 // Specifies whether field trials should be activated (marked as "used"), when 296 // created using |CreateTrialsFromString()|. 297 enum FieldTrialActivationMode { 298 DONT_ACTIVATE_TRIALS, 299 ACTIVATE_TRIALS, 300 }; 301 302 // Define a separator character to use when creating a persistent form of an 303 // instance. This is intended for use as a command line argument, passed to a 304 // second process to mimic our state (i.e., provide the same group name). 305 static const char kPersistentStringSeparator; // Currently a slash. 306 307 // Year that is guaranteed to not be expired when instantiating a field trial 308 // via |FactoryGetFieldTrial()|. Set to two years from the build date. 309 static int kNoExpirationYear; 310 311 // Observer is notified when a FieldTrial's group is selected. 312 class BASE_EXPORT Observer { 313 public: 314 // Notify observers when FieldTrials's group is selected. 315 virtual void OnFieldTrialGroupFinalized(const std::string& trial_name, 316 const std::string& group_name) = 0; 317 318 protected: 319 virtual ~Observer(); 320 }; 321 322 // This singleton holds the global list of registered FieldTrials. 323 // 324 // To support one-time randomized field trials, specify a non-NULL 325 // |entropy_provider| which should be a source of uniformly distributed 326 // entropy values. Takes ownership of |entropy_provider|. If one time 327 // randomization is not desired, pass in NULL for |entropy_provider|. 328 explicit FieldTrialList(const FieldTrial::EntropyProvider* entropy_provider); 329 330 // Destructor Release()'s references to all registered FieldTrial instances. 331 ~FieldTrialList(); 332 333 // Get a FieldTrial instance from the factory. 334 // 335 // |name| is used to register the instance with the FieldTrialList class, 336 // and can be used to find the trial (only one trial can be present for each 337 // name). |default_group_name| is the name of the default group which will 338 // be chosen if none of the subsequent appended groups get to be chosen. 339 // |default_group_number| can receive the group number of the default group as 340 // AppendGroup returns the number of the subsequence groups. |trial_name| and 341 // |default_group_name| may not be empty but |default_group_number| can be 342 // NULL if the value is not needed. 343 // 344 // Group probabilities that are later supplied must sum to less than or equal 345 // to the |total_probability|. Arguments |year|, |month| and |day_of_month| 346 // specify the expiration time. If the build time is after the expiration time 347 // then the field trial reverts to the 'default' group. 348 // 349 // Use this static method to get a startup-randomized FieldTrial or a 350 // previously created forced FieldTrial. 351 static FieldTrial* FactoryGetFieldTrial( 352 const std::string& trial_name, 353 FieldTrial::Probability total_probability, 354 const std::string& default_group_name, 355 const int year, 356 const int month, 357 const int day_of_month, 358 FieldTrial::RandomizationType randomization_type, 359 int* default_group_number); 360 361 // Same as FactoryGetFieldTrial(), but allows specifying a custom seed to be 362 // used on one-time randomized field trials (instead of a hash of the trial 363 // name, which is used otherwise or if |randomization_seed| has value 0). The 364 // |randomization_seed| value (other than 0) should never be the same for two 365 // trials, else this would result in correlated group assignments. 366 // Note: Using a custom randomization seed is only supported by the 367 // PermutedEntropyProvider (which is used when UMA is not enabled). 368 static FieldTrial* FactoryGetFieldTrialWithRandomizationSeed( 369 const std::string& trial_name, 370 FieldTrial::Probability total_probability, 371 const std::string& default_group_name, 372 const int year, 373 const int month, 374 const int day_of_month, 375 FieldTrial::RandomizationType randomization_type, 376 uint32 randomization_seed, 377 int* default_group_number); 378 379 // The Find() method can be used to test to see if a named Trial was already 380 // registered, or to retrieve a pointer to it from the global map. 381 static FieldTrial* Find(const std::string& name); 382 383 // Returns the group number chosen for the named trial, or 384 // FieldTrial::kNotFinalized if the trial does not exist. 385 static int FindValue(const std::string& name); 386 387 // Returns the group name chosen for the named trial, or the 388 // empty string if the trial does not exist. 389 static std::string FindFullName(const std::string& name); 390 391 // Returns true if the named trial has been registered. 392 static bool TrialExists(const std::string& name); 393 394 // Creates a persistent representation of active FieldTrial instances for 395 // resurrection in another process. This allows randomization to be done in 396 // one process, and secondary processes can be synchronized on the result. 397 // The resulting string contains the name and group name pairs of all 398 // registered FieldTrials for which the group has been chosen and externally 399 // observed (via |group()|) and which have not been disabled, with "/" used 400 // to separate all names and to terminate the string. This string is parsed 401 // by |CreateTrialsFromString()|. 402 static void StatesToString(std::string* output); 403 404 // Fills in the supplied vector |active_groups| (which must be empty when 405 // called) with a snapshot of all registered FieldTrials for which the group 406 // has been chosen and externally observed (via |group()|) and which have 407 // not been disabled. 408 static void GetActiveFieldTrialGroups( 409 FieldTrial::ActiveGroups* active_groups); 410 411 // Use a state string (re: StatesToString()) to augment the current list of 412 // field trials to include the supplied trials, and using a 100% probability 413 // for each trial, force them to have the same group string. This is commonly 414 // used in a non-browser process, to carry randomly selected state in a 415 // browser process into this non-browser process, but could also be invoked 416 // through a command line argument to the browser process. The created field 417 // trials are marked as "used" for the purposes of active trial reporting if 418 // |mode| is ACTIVATE_TRIALS. 419 static bool CreateTrialsFromString(const std::string& prior_trials, 420 FieldTrialActivationMode mode); 421 422 // Create a FieldTrial with the given |name| and using 100% probability for 423 // the FieldTrial, force FieldTrial to have the same group string as 424 // |group_name|. This is commonly used in a non-browser process, to carry 425 // randomly selected state in a browser process into this non-browser process. 426 // It returns NULL if there is a FieldTrial that is already registered with 427 // the same |name| but has different finalized group string (|group_name|). 428 static FieldTrial* CreateFieldTrial(const std::string& name, 429 const std::string& group_name); 430 431 // Add an observer to be notified when a field trial is irrevocably committed 432 // to being part of some specific field_group (and hence the group_name is 433 // also finalized for that field_trial). 434 static void AddObserver(Observer* observer); 435 436 // Remove an observer. 437 static void RemoveObserver(Observer* observer); 438 439 // Notify all observers that a group has been finalized for |field_trial|. 440 static void NotifyFieldTrialGroupSelection(FieldTrial* field_trial); 441 442 // Return the number of active field trials. 443 static size_t GetFieldTrialCount(); 444 445 private: 446 // A map from FieldTrial names to the actual instances. 447 typedef std::map<std::string, FieldTrial*> RegistrationList; 448 449 // If one-time randomization is enabled, returns a weak pointer to the 450 // corresponding EntropyProvider. Otherwise, returns NULL. 451 static const FieldTrial::EntropyProvider* 452 GetEntropyProviderForOneTimeRandomization(); 453 454 // Helper function should be called only while holding lock_. 455 FieldTrial* PreLockedFind(const std::string& name); 456 457 // Register() stores a pointer to the given trial in a global map. 458 // This method also AddRef's the indicated trial. 459 // This should always be called after creating a new FieldTrial instance. 460 static void Register(FieldTrial* trial); 461 462 static FieldTrialList* global_; // The singleton of this class. 463 464 // This will tell us if there is an attempt to register a field 465 // trial or check if one-time randomization is enabled without 466 // creating the FieldTrialList. This is not an error, unless a 467 // FieldTrialList is created after that. 468 static bool used_without_global_; 469 470 // Lock for access to registered_. 471 base::Lock lock_; 472 RegistrationList registered_; 473 474 // Entropy provider to be used for one-time randomized field trials. If NULL, 475 // one-time randomization is not supported. 476 scoped_ptr<const FieldTrial::EntropyProvider> entropy_provider_; 477 478 // List of observers to be notified when a group is selected for a FieldTrial. 479 scoped_refptr<ObserverListThreadSafe<Observer> > observer_list_; 480 481 DISALLOW_COPY_AND_ASSIGN(FieldTrialList); 482}; 483 484} // namespace base 485 486#endif // BASE_METRICS_FIELD_TRIAL_H_ 487