1// Copyright 2014 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "components/variations/variations_seed_simulator.h"
6
7#include <map>
8
9#include "base/strings/stringprintf.h"
10#include "components/variations/processed_study.h"
11#include "components/variations/proto/study.pb.h"
12#include "components/variations/variations_associated_data.h"
13#include "testing/gtest/include/gtest/gtest.h"
14
15namespace variations {
16
17namespace {
18
19// An implementation of EntropyProvider that always returns a specific entropy
20// value, regardless of field trial.
21class TestEntropyProvider : public base::FieldTrial::EntropyProvider {
22 public:
23  explicit TestEntropyProvider(double entropy_value)
24      : entropy_value_(entropy_value) {}
25  virtual ~TestEntropyProvider() {}
26
27  // base::FieldTrial::EntropyProvider implementation:
28  virtual double GetEntropyForTrial(const std::string& trial_name,
29                                    uint32 randomization_seed) const OVERRIDE {
30    return entropy_value_;
31  }
32
33 private:
34  const double entropy_value_;
35
36  DISALLOW_COPY_AND_ASSIGN(TestEntropyProvider);
37};
38
39// Creates and activates a single-group field trial with name |trial_name| and
40// group |group_name| and variations |params| (if not NULL).
41void CreateTrial(const std::string& trial_name,
42                 const std::string& group_name,
43                 const std::map<std::string, std::string>* params) {
44  base::FieldTrialList::CreateFieldTrial(trial_name, group_name);
45  if (params != NULL)
46    AssociateVariationParams(trial_name, group_name, *params);
47  base::FieldTrialList::FindFullName(trial_name);
48}
49
50// Creates a study with the given |study_name| and |consistency|.
51Study CreateStudy(const std::string& study_name,
52                  Study_Consistency consistency) {
53  Study study;
54  study.set_name(study_name);
55  study.set_consistency(consistency);
56  return study;
57}
58
59// Adds an experiment to |study| with the specified |experiment_name| and
60// |probability| values and sets it as the study's default experiment.
61Study_Experiment* AddExperiment(const std::string& experiment_name,
62                                int probability,
63                                Study* study) {
64  Study_Experiment* experiment = study->add_experiment();
65  experiment->set_name(experiment_name);
66  experiment->set_probability_weight(probability);
67  study->set_default_experiment_name(experiment_name);
68  return experiment;
69}
70
71// Add an experiment param with |param_name| and |param_value| to |experiment|.
72Study_Experiment_Param* AddExperimentParam(const std::string& param_name,
73                                           const std::string& param_value,
74                                           Study_Experiment* experiment) {
75  Study_Experiment_Param* param = experiment->add_param();
76  param->set_name(param_name);
77  param->set_value(param_value);
78  return param;
79}
80
81}  // namespace
82
83class VariationsSeedSimulatorTest : public ::testing::Test {
84 public:
85  VariationsSeedSimulatorTest() : field_trial_list_(NULL) {
86  }
87
88  virtual ~VariationsSeedSimulatorTest() {
89    // Ensure that the maps are cleared between tests, since they are stored as
90    // process singletons.
91    testing::ClearAllVariationIDs();
92    testing::ClearAllVariationParams();
93  }
94
95  // Uses a VariationsSeedSimulator to simulate the differences between
96  // |studies| and the current field trial state.
97  VariationsSeedSimulator::Result SimulateDifferences(
98      const std::vector<ProcessedStudy>& studies) {
99    TestEntropyProvider provider(0.5);
100    VariationsSeedSimulator seed_simulator(provider);
101    return seed_simulator.ComputeDifferences(studies);
102  }
103
104  // Simulates the differences between |study| and the current field trial
105  // state, returning a string like "1 2 3", where 1 is the number of regular
106  // group changes, 2 is the number of "kill best effort" group changes and 3
107  // is the number of "kill critical" group changes.
108  std::string SimulateStudyDifferences(const Study* study) {
109    std::vector<ProcessedStudy> studies;
110    if (!ProcessedStudy::ValidateAndAppendStudy(study, false, &studies))
111      return "invalid study";
112    return ConvertSimulationResultToString(SimulateDifferences(studies));
113  }
114
115  // Simulates the differences between expired |study| and the current field
116  // trial state, returning a string like "1 2 3", where 1 is the number of
117  // regular group changes, 2 is the number of "kill best effort" group changes
118  // and 3 is the number of "kill critical" group changes.
119  std::string SimulateStudyDifferencesExpired(const Study* study) {
120    std::vector<ProcessedStudy> studies;
121    if (!ProcessedStudy::ValidateAndAppendStudy(study, true, &studies))
122      return "invalid study";
123    if (!studies[0].is_expired())
124      return "not expired";
125    return ConvertSimulationResultToString(SimulateDifferences(studies));
126  }
127
128  // Formats |result| as a string with format "1 2 3", where 1 is the number of
129  // regular group changes, 2 is the number of "kill best effort" group changes
130  // and 3 is the number of "kill critical" group changes.
131  std::string ConvertSimulationResultToString(
132      const VariationsSeedSimulator::Result& result) {
133    return base::StringPrintf("%d %d %d",
134                              result.normal_group_change_count,
135                              result.kill_best_effort_group_change_count,
136                              result.kill_critical_group_change_count);
137  }
138
139 private:
140  base::FieldTrialList field_trial_list_;
141
142  DISALLOW_COPY_AND_ASSIGN(VariationsSeedSimulatorTest);
143};
144
145TEST_F(VariationsSeedSimulatorTest, PermanentNoChanges) {
146  CreateTrial("A", "B", NULL);
147
148  std::vector<ProcessedStudy> processed_studies;
149  Study study = CreateStudy("A", Study_Consistency_PERMANENT);
150  Study_Experiment* experiment = AddExperiment("B", 100, &study);
151
152  EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
153
154  experiment->set_type(Study_Experiment_Type_NORMAL);
155  EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
156  experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
157  EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
158  experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
159  EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
160  experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
161  EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
162}
163
164TEST_F(VariationsSeedSimulatorTest, PermanentGroupChange) {
165  CreateTrial("A", "B", NULL);
166
167  Study study = CreateStudy("A", Study_Consistency_PERMANENT);
168  Study_Experiment* experiment = AddExperiment("C", 100, &study);
169
170  EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
171
172  // Changing "C" group type should not affect the type of change. (Since the
173  // type is evaluated for the "old" group.)
174  experiment->set_type(Study_Experiment_Type_NORMAL);
175  EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
176  experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
177  EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
178  experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
179  EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
180  experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
181  EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
182}
183
184TEST_F(VariationsSeedSimulatorTest, PermanentExpired) {
185  CreateTrial("A", "B", NULL);
186
187  Study study = CreateStudy("A", Study_Consistency_PERMANENT);
188  Study_Experiment* experiment = AddExperiment("B", 1, &study);
189  AddExperiment("C", 0, &study);
190
191  // There should be a difference because the study is expired, which should
192  // result in the default group "C" being chosen.
193  EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
194
195  experiment->set_type(Study_Experiment_Type_NORMAL);
196  EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
197  experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
198  EXPECT_EQ("0 0 0", SimulateStudyDifferencesExpired(&study));
199  experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
200  EXPECT_EQ("0 1 0", SimulateStudyDifferencesExpired(&study));
201  experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
202  EXPECT_EQ("0 0 1", SimulateStudyDifferencesExpired(&study));
203}
204
205TEST_F(VariationsSeedSimulatorTest, SessionRandomized) {
206  CreateTrial("A", "B", NULL);
207
208  Study study = CreateStudy("A", Study_Consistency_SESSION);
209  Study_Experiment* experiment = AddExperiment("B", 1, &study);
210  AddExperiment("C", 1, &study);
211  AddExperiment("D", 1, &study);
212
213  // There should be no differences, since a session randomized study can result
214  // in any of the groups being chosen on startup.
215  EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
216
217  experiment->set_type(Study_Experiment_Type_NORMAL);
218  EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
219  experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
220  EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
221  experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
222  EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
223  experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
224  EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
225}
226
227TEST_F(VariationsSeedSimulatorTest, SessionRandomizedGroupRemoved) {
228  CreateTrial("A", "B", NULL);
229
230  Study study = CreateStudy("A", Study_Consistency_SESSION);
231  AddExperiment("C", 1, &study);
232  AddExperiment("D", 1, &study);
233
234  // There should be a difference since there is no group "B" in the new config.
235  EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
236}
237
238TEST_F(VariationsSeedSimulatorTest, SessionRandomizedGroupProbabilityZero) {
239  CreateTrial("A", "B", NULL);
240
241  Study study = CreateStudy("A", Study_Consistency_SESSION);
242  Study_Experiment* experiment = AddExperiment("B", 0, &study);
243  AddExperiment("C", 1, &study);
244  AddExperiment("D", 1, &study);
245
246  // There should be a difference since group "B" has probability 0.
247  EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
248
249  experiment->set_type(Study_Experiment_Type_NORMAL);
250  EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
251  experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
252  EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
253  experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
254  EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
255  experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
256  EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
257}
258
259TEST_F(VariationsSeedSimulatorTest, SessionRandomizedExpired) {
260  CreateTrial("A", "B", NULL);
261
262  Study study = CreateStudy("A", Study_Consistency_SESSION);
263  Study_Experiment* experiment = AddExperiment("B", 1, &study);
264  AddExperiment("C", 1, &study);
265  AddExperiment("D", 1, &study);
266
267  // There should be a difference because the study is expired, which should
268  // result in the default group "D" being chosen.
269  EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
270
271  experiment->set_type(Study_Experiment_Type_NORMAL);
272  EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
273  experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
274  EXPECT_EQ("0 0 0", SimulateStudyDifferencesExpired(&study));
275  experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
276  EXPECT_EQ("0 1 0", SimulateStudyDifferencesExpired(&study));
277  experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
278  EXPECT_EQ("0 0 1", SimulateStudyDifferencesExpired(&study));
279}
280
281TEST_F(VariationsSeedSimulatorTest, ParamsUnchanged) {
282  std::map<std::string, std::string> params;
283  params["p1"] = "x";
284  params["p2"] = "y";
285  params["p3"] = "z";
286  CreateTrial("A", "B", &params);
287
288  std::vector<ProcessedStudy> processed_studies;
289  Study study = CreateStudy("A", Study_Consistency_PERMANENT);
290  Study_Experiment* experiment = AddExperiment("B", 100, &study);
291  AddExperimentParam("p2", "y", experiment);
292  AddExperimentParam("p1", "x", experiment);
293  AddExperimentParam("p3", "z", experiment);
294
295  EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
296
297  experiment->set_type(Study_Experiment_Type_NORMAL);
298  EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
299  experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
300  EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
301  experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
302  EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
303  experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
304  EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
305}
306
307TEST_F(VariationsSeedSimulatorTest, ParamsChanged) {
308  std::map<std::string, std::string> params;
309  params["p1"] = "x";
310  params["p2"] = "y";
311  params["p3"] = "z";
312  CreateTrial("A", "B", &params);
313
314  std::vector<ProcessedStudy> processed_studies;
315  Study study = CreateStudy("A", Study_Consistency_PERMANENT);
316  Study_Experiment* experiment = AddExperiment("B", 100, &study);
317  AddExperimentParam("p2", "test", experiment);
318  AddExperimentParam("p1", "x", experiment);
319  AddExperimentParam("p3", "z", experiment);
320
321  // The param lists differ.
322  EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
323
324  experiment->set_type(Study_Experiment_Type_NORMAL);
325  EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
326  experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
327  EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
328  experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
329  EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
330  experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
331  EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
332}
333
334TEST_F(VariationsSeedSimulatorTest, ParamsRemoved) {
335  std::map<std::string, std::string> params;
336  params["p1"] = "x";
337  params["p2"] = "y";
338  params["p3"] = "z";
339  CreateTrial("A", "B", &params);
340
341  std::vector<ProcessedStudy> processed_studies;
342  Study study = CreateStudy("A", Study_Consistency_PERMANENT);
343  Study_Experiment* experiment = AddExperiment("B", 100, &study);
344
345  // The current group has params, but the new config doesn't have any.
346  EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
347
348  experiment->set_type(Study_Experiment_Type_NORMAL);
349  EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
350  experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
351  EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
352  experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
353  EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
354  experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
355  EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
356}
357
358TEST_F(VariationsSeedSimulatorTest, ParamsAdded) {
359  CreateTrial("A", "B", NULL);
360
361  std::vector<ProcessedStudy> processed_studies;
362  Study study = CreateStudy("A", Study_Consistency_PERMANENT);
363  Study_Experiment* experiment = AddExperiment("B", 100, &study);
364  AddExperimentParam("p2", "y", experiment);
365  AddExperimentParam("p1", "x", experiment);
366  AddExperimentParam("p3", "z", experiment);
367
368  // The current group has no params, but the config has added some.
369  EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
370
371  experiment->set_type(Study_Experiment_Type_NORMAL);
372  EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
373  experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
374  EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
375  experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
376  EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
377  experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
378  EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
379}
380
381}  // namespace variations
382