1/*
2 * Copyright 2018 Google Inc.
3 *
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
6 */
7
8package main
9
10import (
11	"bytes"
12	"encoding/json"
13	"flag"
14	"fmt"
15	"net/http"
16	"os"
17	"os/exec"
18	"sort"
19	"strconv"
20	"strings"
21	"syscall"
22	"time"
23
24	gstorage "google.golang.org/api/storage/v1"
25
26	"go.skia.org/infra/go/auth"
27	"go.skia.org/infra/go/common"
28	"go.skia.org/infra/go/sklog"
29	"go.skia.org/infra/go/util"
30	"go.skia.org/infra/golden/go/tsuite"
31)
32
33// TODO(stephana): Convert the hard coded whitelist to a command line flag that
34// loads a file with the whitelisted devices and versions. Make sure to include
35// human readable names for the devices.
36
37var (
38	// WHITELIST_DEV_IDS contains a mapping from the device id to the list of
39	// Android API versions that we should run agains. Usually this will be the
40	// latest version. To see available devices and version run with
41	// --dryrun flag or run '$ gcloud firebase test android models list'
42
43	WHITELIST_DEV_IDS = map[string][]string{
44		"A0001": {"22"},
45		// "E5803":       {"22"},    deprecated
46		// "F5121":       {"23"},    deprecated
47		"G8142":      {"25"},
48		"HWMHA":      {"24"},
49		"SH-04H":     {"23"},
50		"athene":     {"23"},
51		"athene_f":   {"23"},
52		"hammerhead": {"23"},
53		"harpia":     {"23"},
54		"hero2lte":   {"23"},
55		"herolte":    {"24"},
56		"j1acevelte": {"22"},
57		"j5lte":      {"23"},
58		"j7xelte":    {"23"},
59		"lucye":      {"24"},
60		// "mako":        {"22"},   deprecated
61		"osprey_umts": {"22"},
62		// "p1":          {"22"},   deprecated
63		"sailfish": {"26"},
64		"shamu":    {"23"},
65		"trelte":   {"22"},
66		"zeroflte": {"22"},
67		"zerolte":  {"22"},
68	}
69)
70
71const (
72	META_DATA_FILENAME = "meta.json"
73)
74
75// Command line flags.
76var (
77	serviceAccountFile = flag.String("service_account_file", "", "Credentials file for service account.")
78	dryRun             = flag.Bool("dryrun", false, "Print out the command and quit without triggering tests.")
79	minAPIVersion      = flag.Int("min_api", 22, "Minimum API version required by device.")
80	maxAPIVersion      = flag.Int("max_api", 23, "Maximum API version required by device.")
81)
82
83const (
84	RUN_TESTS_TEMPLATE = `gcloud beta firebase test android run
85	--type=game-loop
86	--app=%s
87	--results-bucket=%s
88	--results-dir=%s
89	--directories-to-pull=/sdcard/Android/data/org.skia.skqp
90	--timeout 30m
91	%s
92`
93	MODEL_VERSION_TMPL   = "--device model=%s,version=%s,orientation=portrait"
94	RESULT_BUCKET        = "skia-firebase-test-lab"
95	RESULT_DIR_TMPL      = "testruns/%s/%s"
96	RUN_ID_TMPL          = "testrun-%d"
97	CMD_AVAILABE_DEVICES = "gcloud firebase test android models list --format json"
98)
99
100func main() {
101	common.Init()
102
103	// Get the apk.
104	args := flag.Args()
105	apk_path := args[0]
106
107	// Make sure we can get the service account client.
108	client, err := auth.NewJWTServiceAccountClient("", *serviceAccountFile, nil, gstorage.CloudPlatformScope, "https://www.googleapis.com/auth/userinfo.email")
109	if err != nil {
110		sklog.Fatalf("Failed to authenticate service account: %s. Run 'get_service_account' to obtain a service account file.", err)
111	}
112
113	// Get list of all available devices.
114	devices, ignoredDevices, err := getAvailableDevices(WHITELIST_DEV_IDS, *minAPIVersion, *maxAPIVersion)
115	if err != nil {
116		sklog.Fatalf("Unable to retrieve available devices: %s", err)
117	}
118	sklog.Infof("---")
119	sklog.Infof("Selected devices:")
120	logDevices(devices)
121
122	if err := runTests(apk_path, devices, ignoredDevices, client, *dryRun); err != nil {
123		sklog.Fatalf("Error triggering tests on Firebase: %s", err)
124	}
125}
126
127// getAvailableDevices is given a whitelist. It queries Firebase Testlab for all
128// available devices and then returns a list of devices to be tested and the list
129// of ignored devices.
130func getAvailableDevices(whiteList map[string][]string, minAPIVersion, maxAPIVersion int) ([]*tsuite.DeviceVersions, []*tsuite.DeviceVersions, error) {
131	// Get the list of all devices in JSON format from Firebase testlab.
132	var buf bytes.Buffer
133	cmd := parseCommand(CMD_AVAILABE_DEVICES)
134	cmd.Stdout = &buf
135	cmd.Stderr = os.Stdout
136	if err := cmd.Run(); err != nil {
137		return nil, nil, err
138	}
139
140	// Unmarshal the result.
141	foundDevices := []*tsuite.FirebaseDevice{}
142	bufBytes := buf.Bytes()
143	if err := json.Unmarshal(bufBytes, &foundDevices); err != nil {
144		return nil, nil, sklog.FmtErrorf("Unmarshal of device information failed: %s \nJSON Input: %s\n", err, string(bufBytes))
145	}
146
147	// iterate over the available devices and partition them.
148	allDevices := make([]*tsuite.DeviceVersions, 0, len(foundDevices))
149	ret := make([]*tsuite.DeviceVersions, 0, len(foundDevices))
150	ignored := make([]*tsuite.DeviceVersions, 0, len(foundDevices))
151	for _, dev := range foundDevices {
152		// Filter out all the virtual devices.
153		if dev.Form == "PHYSICAL" {
154			// Only include devices that are on the whitelist and have versions defined.
155			if foundVersions, ok := whiteList[dev.ID]; ok && (len(foundVersions) > 0) {
156				versionSet := util.NewStringSet(dev.VersionIDs)
157				reqVersions := util.NewStringSet(filterVersions(foundVersions, minAPIVersion, maxAPIVersion))
158				whiteListVersions := versionSet.Intersect(reqVersions).Keys()
159				ignoredVersions := versionSet.Complement(reqVersions).Keys()
160				sort.Strings(whiteListVersions)
161				sort.Strings(ignoredVersions)
162				ret = append(ret, &tsuite.DeviceVersions{Device: dev, Versions: whiteListVersions})
163				ignored = append(ignored, &tsuite.DeviceVersions{Device: dev, Versions: ignoredVersions})
164			} else {
165				ignored = append(ignored, &tsuite.DeviceVersions{Device: dev, Versions: dev.VersionIDs})
166			}
167			allDevices = append(allDevices, &tsuite.DeviceVersions{Device: dev, Versions: dev.VersionIDs})
168		}
169	}
170
171	sklog.Infof("All devices:")
172	logDevices(allDevices)
173
174	return ret, ignored, nil
175}
176
177// filterVersions returns the elements in versionIDs where minVersion <= element <= maxVersion.
178func filterVersions(versionIDs []string, minVersion, maxVersion int) []string {
179	ret := make([]string, 0, len(versionIDs))
180	for _, versionID := range versionIDs {
181		id, err := strconv.Atoi(versionID)
182		if err != nil {
183			sklog.Fatalf("Error parsing version id '%s': %s", versionID, err)
184		}
185		if (id >= minVersion) && (id <= maxVersion) {
186			ret = append(ret, versionID)
187		}
188	}
189	return ret
190}
191
192// runTests runs the given apk on the given list of devices.
193func runTests(apk_path string, devices, ignoredDevices []*tsuite.DeviceVersions, client *http.Client, dryRun bool) error {
194	// Get the model-version we want to test. Assume on average each model has 5 supported versions.
195	modelSelectors := make([]string, 0, len(devices)*5)
196	for _, devRec := range devices {
197		for _, version := range devRec.Versions {
198			modelSelectors = append(modelSelectors, fmt.Sprintf(MODEL_VERSION_TMPL, devRec.Device.ID, version))
199		}
200	}
201
202	now := time.Now()
203	nowMs := now.UnixNano() / int64(time.Millisecond)
204	runID := fmt.Sprintf(RUN_ID_TMPL, nowMs)
205	resultsDir := fmt.Sprintf(RESULT_DIR_TMPL, now.Format("2006/01/02/15"), runID)
206	cmdStr := fmt.Sprintf(RUN_TESTS_TEMPLATE, apk_path, RESULT_BUCKET, resultsDir, strings.Join(modelSelectors, "\n"))
207	cmdStr = strings.TrimSpace(strings.Replace(cmdStr, "\n", " ", -1))
208
209	// Run the command.
210	cmd := parseCommand(cmdStr)
211	cmd.Stdout = os.Stdout
212	cmd.Stderr = os.Stdout
213	exitCode := 0
214
215	if dryRun {
216		fmt.Printf("[dry run]: Would have run this command: %s\n", cmdStr)
217		return nil
218	}
219
220	if err := cmd.Run(); err != nil {
221		// Get the exit code.
222		if exitError, ok := err.(*exec.ExitError); ok {
223			ws := exitError.Sys().(syscall.WaitStatus)
224			exitCode = ws.ExitStatus()
225		}
226		sklog.Errorf("Error running tests: %s", err)
227		sklog.Errorf("Exit code: %d", exitCode)
228
229		// Exit code 10 means triggering on Testlab succeeded, but but some of the
230		// runs on devices failed. We consider it a success for this script.
231		if exitCode != 10 {
232			return err
233		}
234	}
235
236	// Store the result in a meta json file.
237	meta := &tsuite.TestRunMeta{
238		ID:             runID,
239		TS:             nowMs,
240		Devices:        devices,
241		IgnoredDevices: ignoredDevices,
242		ExitCode:       exitCode,
243	}
244
245	meta.WriteToGCS(RESULT_BUCKET, resultsDir+"/"+META_DATA_FILENAME, client)
246	return nil
247}
248
249// logDevices logs the given list of devices.
250func logDevices(devices []*tsuite.DeviceVersions) {
251	sklog.Infof("Found %d devices.", len(devices))
252	for _, dev := range devices {
253		sklog.Infof("%-15s %-30s %v / %v", dev.Device.ID, dev.Device.Name, dev.Device.VersionIDs, dev.Versions)
254	}
255}
256
257// parseCommad parses a command line and wraps it in an exec.Command instance.
258func parseCommand(cmdStr string) *exec.Cmd {
259	cmdArgs := strings.Split(strings.TrimSpace(cmdStr), " ")
260	for idx := range cmdArgs {
261		cmdArgs[idx] = strings.TrimSpace(cmdArgs[idx])
262	}
263	return exec.Command(cmdArgs[0], cmdArgs[1:]...)
264}
265