1/* Copyright (c) 2015, Google Inc.
2 *
3 * Permission to use, copy, modify, and/or distribute this software for any
4 * purpose with or without fee is hereby granted, provided that the above
5 * copyright notice and this permission notice appear in all copies.
6 *
7 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
8 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
9 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
10 * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
11 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
12 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
13 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
14
15package main
16
17import (
18	"bytes"
19	"encoding/json"
20	"flag"
21	"fmt"
22	"os"
23	"os/exec"
24	"path"
25	"strconv"
26	"strings"
27	"syscall"
28	"time"
29)
30
31// TODO(davidben): Link tests with the malloc shim and port -malloc-test to this runner.
32
33var (
34	useValgrind     = flag.Bool("valgrind", false, "If true, run code under valgrind")
35	useGDB          = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb")
36	buildDir        = flag.String("build-dir", "build", "The build directory to run the tests from.")
37	jsonOutput      = flag.String("json-output", "", "The file to output JSON results to.")
38	mallocTest      = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.")
39	mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask each test to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.")
40)
41
42type test []string
43
44var tests = []test{
45	{"crypto/base64/base64_test"},
46	{"crypto/bio/bio_test"},
47	{"crypto/bn/bn_test"},
48	{"crypto/bytestring/bytestring_test"},
49	{"crypto/cipher/aead_test", "aes-128-gcm", "crypto/cipher/test/aes_128_gcm_tests.txt"},
50	{"crypto/cipher/aead_test", "aes-128-key-wrap", "crypto/cipher/test/aes_128_key_wrap_tests.txt"},
51	{"crypto/cipher/aead_test", "aes-256-gcm", "crypto/cipher/test/aes_256_gcm_tests.txt"},
52	{"crypto/cipher/aead_test", "aes-256-key-wrap", "crypto/cipher/test/aes_256_key_wrap_tests.txt"},
53	{"crypto/cipher/aead_test", "chacha20-poly1305", "crypto/cipher/test/chacha20_poly1305_tests.txt"},
54	{"crypto/cipher/aead_test", "rc4-md5-tls", "crypto/cipher/test/rc4_md5_tls_tests.txt"},
55	{"crypto/cipher/aead_test", "rc4-sha1-tls", "crypto/cipher/test/rc4_sha1_tls_tests.txt"},
56	{"crypto/cipher/aead_test", "aes-128-cbc-sha1-tls", "crypto/cipher/test/aes_128_cbc_sha1_tls_tests.txt"},
57	{"crypto/cipher/aead_test", "aes-128-cbc-sha1-tls-implicit-iv", "crypto/cipher/test/aes_128_cbc_sha1_tls_implicit_iv_tests.txt"},
58	{"crypto/cipher/aead_test", "aes-128-cbc-sha256-tls", "crypto/cipher/test/aes_128_cbc_sha256_tls_tests.txt"},
59	{"crypto/cipher/aead_test", "aes-256-cbc-sha1-tls", "crypto/cipher/test/aes_256_cbc_sha1_tls_tests.txt"},
60	{"crypto/cipher/aead_test", "aes-256-cbc-sha1-tls-implicit-iv", "crypto/cipher/test/aes_256_cbc_sha1_tls_implicit_iv_tests.txt"},
61	{"crypto/cipher/aead_test", "aes-256-cbc-sha256-tls", "crypto/cipher/test/aes_256_cbc_sha256_tls_tests.txt"},
62	{"crypto/cipher/aead_test", "aes-256-cbc-sha384-tls", "crypto/cipher/test/aes_256_cbc_sha384_tls_tests.txt"},
63	{"crypto/cipher/aead_test", "des-ede3-cbc-sha1-tls", "crypto/cipher/test/des_ede3_cbc_sha1_tls_tests.txt"},
64	{"crypto/cipher/aead_test", "des-ede3-cbc-sha1-tls-implicit-iv", "crypto/cipher/test/des_ede3_cbc_sha1_tls_implicit_iv_tests.txt"},
65	{"crypto/cipher/aead_test", "rc4-md5-ssl3", "crypto/cipher/test/rc4_md5_ssl3_tests.txt"},
66	{"crypto/cipher/aead_test", "rc4-sha1-ssl3", "crypto/cipher/test/rc4_sha1_ssl3_tests.txt"},
67	{"crypto/cipher/aead_test", "aes-128-cbc-sha1-ssl3", "crypto/cipher/test/aes_128_cbc_sha1_ssl3_tests.txt"},
68	{"crypto/cipher/aead_test", "aes-256-cbc-sha1-ssl3", "crypto/cipher/test/aes_256_cbc_sha1_ssl3_tests.txt"},
69	{"crypto/cipher/aead_test", "des-ede3-cbc-sha1-ssl3", "crypto/cipher/test/des_ede3_cbc_sha1_ssl3_tests.txt"},
70	{"crypto/cipher/aead_test", "aes-128-ctr-hmac-sha256", "crypto/cipher/test/aes_128_ctr_hmac_sha256.txt"},
71	{"crypto/cipher/aead_test", "aes-256-ctr-hmac-sha256", "crypto/cipher/test/aes_256_ctr_hmac_sha256.txt"},
72	{"crypto/cipher/cipher_test", "crypto/cipher/test/cipher_test.txt"},
73	{"crypto/cmac/cmac_test"},
74	{"crypto/constant_time_test"},
75	{"crypto/dh/dh_test"},
76	{"crypto/digest/digest_test"},
77	{"crypto/dsa/dsa_test"},
78	{"crypto/ec/ec_test"},
79	{"crypto/ec/example_mul"},
80	{"crypto/ecdsa/ecdsa_test"},
81	{"crypto/err/err_test"},
82	{"crypto/evp/evp_extra_test"},
83	{"crypto/evp/evp_test", "crypto/evp/evp_tests.txt"},
84	{"crypto/evp/evp_test", "crypto/hmac/hmac_tests.txt"},
85	{"crypto/evp/pbkdf_test"},
86	{"crypto/hkdf/hkdf_test"},
87	{"crypto/hmac/hmac_test", "crypto/hmac/hmac_tests.txt"},
88	{"crypto/lhash/lhash_test"},
89	{"crypto/modes/gcm_test"},
90	{"crypto/pkcs8/pkcs12_test"},
91	{"crypto/refcount_test"},
92	{"crypto/rsa/rsa_test"},
93	{"crypto/thread_test"},
94	{"crypto/x509/pkcs7_test"},
95	{"crypto/x509v3/tab_test"},
96	{"crypto/x509v3/v3name_test"},
97	{"ssl/pqueue/pqueue_test"},
98	{"ssl/ssl_test"},
99}
100
101// testOutput is a representation of Chromium's JSON test result format. See
102// https://www.chromium.org/developers/the-json-test-results-format
103type testOutput struct {
104	Version           int                   `json:"version"`
105	Interrupted       bool                  `json:"interrupted"`
106	PathDelimiter     string                `json:"path_delimiter"`
107	SecondsSinceEpoch float64               `json:"seconds_since_epoch"`
108	NumFailuresByType map[string]int        `json:"num_failures_by_type"`
109	Tests             map[string]testResult `json:"tests"`
110}
111
112type testResult struct {
113	Actual       string `json:"actual"`
114	Expected     string `json:"expected"`
115	IsUnexpected bool   `json:"is_unexpected"`
116}
117
118func newTestOutput() *testOutput {
119	return &testOutput{
120		Version:           3,
121		PathDelimiter:     ".",
122		SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond),
123		NumFailuresByType: make(map[string]int),
124		Tests:             make(map[string]testResult),
125	}
126}
127
128func (t *testOutput) addResult(name, result string) {
129	if _, found := t.Tests[name]; found {
130		panic(name)
131	}
132	t.Tests[name] = testResult{
133		Actual:       result,
134		Expected:     "PASS",
135		IsUnexpected: result != "PASS",
136	}
137	t.NumFailuresByType[result]++
138}
139
140func (t *testOutput) writeTo(name string) error {
141	file, err := os.Create(name)
142	if err != nil {
143		return err
144	}
145	defer file.Close()
146	out, err := json.MarshalIndent(t, "", "  ")
147	if err != nil {
148		return err
149	}
150	_, err = file.Write(out)
151	return err
152}
153
154func valgrindOf(dbAttach bool, path string, args ...string) *exec.Cmd {
155	valgrindArgs := []string{"--error-exitcode=99", "--track-origins=yes", "--leak-check=full"}
156	if dbAttach {
157		valgrindArgs = append(valgrindArgs, "--db-attach=yes", "--db-command=xterm -e gdb -nw %f %p")
158	}
159	valgrindArgs = append(valgrindArgs, path)
160	valgrindArgs = append(valgrindArgs, args...)
161
162	return exec.Command("valgrind", valgrindArgs...)
163}
164
165func gdbOf(path string, args ...string) *exec.Cmd {
166	xtermArgs := []string{"-e", "gdb", "--args"}
167	xtermArgs = append(xtermArgs, path)
168	xtermArgs = append(xtermArgs, args...)
169
170	return exec.Command("xterm", xtermArgs...)
171}
172
173type moreMallocsError struct{}
174
175func (moreMallocsError) Error() string {
176	return "child process did not exhaust all allocation calls"
177}
178
179var errMoreMallocs = moreMallocsError{}
180
181func runTestOnce(test test, mallocNumToFail int64) (passed bool, err error) {
182	prog := path.Join(*buildDir, test[0])
183	args := test[1:]
184	var cmd *exec.Cmd
185	if *useValgrind {
186		cmd = valgrindOf(false, prog, args...)
187	} else if *useGDB {
188		cmd = gdbOf(prog, args...)
189	} else {
190		cmd = exec.Command(prog, args...)
191	}
192	var stdoutBuf bytes.Buffer
193	var stderrBuf bytes.Buffer
194	cmd.Stdout = &stdoutBuf
195	cmd.Stderr = &stderrBuf
196	if mallocNumToFail >= 0 {
197		cmd.Env = os.Environ()
198		cmd.Env = append(cmd.Env, "MALLOC_NUMBER_TO_FAIL="+strconv.FormatInt(mallocNumToFail, 10))
199		if *mallocTestDebug {
200			cmd.Env = append(cmd.Env, "MALLOC_ABORT_ON_FAIL=1")
201		}
202		cmd.Env = append(cmd.Env, "_MALLOC_CHECK=1")
203	}
204
205	if err := cmd.Start(); err != nil {
206		return false, err
207	}
208	if err := cmd.Wait(); err != nil {
209		if exitError, ok := err.(*exec.ExitError); ok {
210			if exitError.Sys().(syscall.WaitStatus).ExitStatus() == 88 {
211				return false, errMoreMallocs
212			}
213		}
214		fmt.Print(string(stderrBuf.Bytes()))
215		return false, err
216	}
217	fmt.Print(string(stderrBuf.Bytes()))
218
219	// Account for Windows line-endings.
220	stdout := bytes.Replace(stdoutBuf.Bytes(), []byte("\r\n"), []byte("\n"), -1)
221
222	if bytes.HasSuffix(stdout, []byte("PASS\n")) &&
223		(len(stdout) == 5 || stdout[len(stdout)-6] == '\n') {
224		return true, nil
225	}
226	return false, nil
227}
228
229func runTest(test test) (bool, error) {
230	if *mallocTest < 0 {
231		return runTestOnce(test, -1)
232	}
233
234	for mallocNumToFail := int64(*mallocTest); ; mallocNumToFail++ {
235		if passed, err := runTestOnce(test, mallocNumToFail); err != errMoreMallocs {
236			if err != nil {
237				err = fmt.Errorf("at malloc %d: %s", mallocNumToFail, err)
238			}
239			return passed, err
240		}
241	}
242}
243
244// shortTestName returns the short name of a test. Except for evp_test, it
245// assumes that any argument which ends in .txt is a path to a data file and not
246// relevant to the test's uniqueness.
247func shortTestName(test test) string {
248	var args []string
249	for _, arg := range test {
250		if test[0] == "crypto/evp/evp_test" || !strings.HasSuffix(arg, ".txt") {
251			args = append(args, arg)
252		}
253	}
254	return strings.Join(args, " ")
255}
256
257func main() {
258	flag.Parse()
259
260	testOutput := newTestOutput()
261	var failed []test
262	for _, test := range tests {
263		fmt.Printf("%s\n", strings.Join([]string(test), " "))
264
265		name := shortTestName(test)
266		passed, err := runTest(test)
267		if err != nil {
268			fmt.Printf("%s failed to complete: %s\n", test[0], err)
269			failed = append(failed, test)
270			testOutput.addResult(name, "CRASHED")
271		} else if !passed {
272			fmt.Printf("%s failed to print PASS on the last line.\n", test[0])
273			failed = append(failed, test)
274			testOutput.addResult(name, "FAIL")
275		} else {
276			testOutput.addResult(name, "PASS")
277		}
278	}
279
280	if *jsonOutput != "" {
281		if err := testOutput.writeTo(*jsonOutput); err != nil {
282			fmt.Fprintf(os.Stderr, "Error: %s\n", err)
283		}
284	}
285
286	if len(failed) > 0 {
287		fmt.Printf("\n%d of %d tests failed:\n", len(failed), len(tests))
288		for _, test := range failed {
289			fmt.Printf("\t%s\n", strings.Join([]string(test), " "))
290		}
291		os.Exit(1)
292	}
293
294	fmt.Printf("\nAll tests passed!\n")
295}
296