1/* Copyright (c) 2015, Google Inc. 2 * 3 * Permission to use, copy, modify, and/or distribute this software for any 4 * purpose with or without fee is hereby granted, provided that the above 5 * copyright notice and this permission notice appear in all copies. 6 * 7 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 10 * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION 12 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 13 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ 14 15package main 16 17import ( 18 "bytes" 19 "encoding/json" 20 "flag" 21 "fmt" 22 "os" 23 "os/exec" 24 "path" 25 "strconv" 26 "strings" 27 "syscall" 28 "time" 29) 30 31// TODO(davidben): Link tests with the malloc shim and port -malloc-test to this runner. 32 33var ( 34 useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind") 35 useGDB = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb") 36 buildDir = flag.String("build-dir", "build", "The build directory to run the tests from.") 37 jsonOutput = flag.String("json-output", "", "The file to output JSON results to.") 38 mallocTest = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.") 39 mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask each test to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.") 40) 41 42type test []string 43 44var tests = []test{ 45 {"crypto/base64/base64_test"}, 46 {"crypto/bio/bio_test"}, 47 {"crypto/bn/bn_test"}, 48 {"crypto/bytestring/bytestring_test"}, 49 {"crypto/cipher/aead_test", "aes-128-gcm", "crypto/cipher/test/aes_128_gcm_tests.txt"}, 50 {"crypto/cipher/aead_test", "aes-128-key-wrap", "crypto/cipher/test/aes_128_key_wrap_tests.txt"}, 51 {"crypto/cipher/aead_test", "aes-256-gcm", "crypto/cipher/test/aes_256_gcm_tests.txt"}, 52 {"crypto/cipher/aead_test", "aes-256-key-wrap", "crypto/cipher/test/aes_256_key_wrap_tests.txt"}, 53 {"crypto/cipher/aead_test", "chacha20-poly1305", "crypto/cipher/test/chacha20_poly1305_tests.txt"}, 54 {"crypto/cipher/aead_test", "rc4-md5-tls", "crypto/cipher/test/rc4_md5_tls_tests.txt"}, 55 {"crypto/cipher/aead_test", "rc4-sha1-tls", "crypto/cipher/test/rc4_sha1_tls_tests.txt"}, 56 {"crypto/cipher/aead_test", "aes-128-cbc-sha1-tls", "crypto/cipher/test/aes_128_cbc_sha1_tls_tests.txt"}, 57 {"crypto/cipher/aead_test", "aes-128-cbc-sha1-tls-implicit-iv", "crypto/cipher/test/aes_128_cbc_sha1_tls_implicit_iv_tests.txt"}, 58 {"crypto/cipher/aead_test", "aes-128-cbc-sha256-tls", "crypto/cipher/test/aes_128_cbc_sha256_tls_tests.txt"}, 59 {"crypto/cipher/aead_test", "aes-256-cbc-sha1-tls", "crypto/cipher/test/aes_256_cbc_sha1_tls_tests.txt"}, 60 {"crypto/cipher/aead_test", "aes-256-cbc-sha1-tls-implicit-iv", "crypto/cipher/test/aes_256_cbc_sha1_tls_implicit_iv_tests.txt"}, 61 {"crypto/cipher/aead_test", "aes-256-cbc-sha256-tls", "crypto/cipher/test/aes_256_cbc_sha256_tls_tests.txt"}, 62 {"crypto/cipher/aead_test", "aes-256-cbc-sha384-tls", "crypto/cipher/test/aes_256_cbc_sha384_tls_tests.txt"}, 63 {"crypto/cipher/aead_test", "des-ede3-cbc-sha1-tls", "crypto/cipher/test/des_ede3_cbc_sha1_tls_tests.txt"}, 64 {"crypto/cipher/aead_test", "des-ede3-cbc-sha1-tls-implicit-iv", "crypto/cipher/test/des_ede3_cbc_sha1_tls_implicit_iv_tests.txt"}, 65 {"crypto/cipher/aead_test", "rc4-md5-ssl3", "crypto/cipher/test/rc4_md5_ssl3_tests.txt"}, 66 {"crypto/cipher/aead_test", "rc4-sha1-ssl3", "crypto/cipher/test/rc4_sha1_ssl3_tests.txt"}, 67 {"crypto/cipher/aead_test", "aes-128-cbc-sha1-ssl3", "crypto/cipher/test/aes_128_cbc_sha1_ssl3_tests.txt"}, 68 {"crypto/cipher/aead_test", "aes-256-cbc-sha1-ssl3", "crypto/cipher/test/aes_256_cbc_sha1_ssl3_tests.txt"}, 69 {"crypto/cipher/aead_test", "des-ede3-cbc-sha1-ssl3", "crypto/cipher/test/des_ede3_cbc_sha1_ssl3_tests.txt"}, 70 {"crypto/cipher/aead_test", "aes-128-ctr-hmac-sha256", "crypto/cipher/test/aes_128_ctr_hmac_sha256.txt"}, 71 {"crypto/cipher/aead_test", "aes-256-ctr-hmac-sha256", "crypto/cipher/test/aes_256_ctr_hmac_sha256.txt"}, 72 {"crypto/cipher/cipher_test", "crypto/cipher/test/cipher_test.txt"}, 73 {"crypto/cmac/cmac_test"}, 74 {"crypto/constant_time_test"}, 75 {"crypto/dh/dh_test"}, 76 {"crypto/digest/digest_test"}, 77 {"crypto/dsa/dsa_test"}, 78 {"crypto/ec/ec_test"}, 79 {"crypto/ec/example_mul"}, 80 {"crypto/ecdsa/ecdsa_test"}, 81 {"crypto/err/err_test"}, 82 {"crypto/evp/evp_extra_test"}, 83 {"crypto/evp/evp_test", "crypto/evp/evp_tests.txt"}, 84 {"crypto/evp/evp_test", "crypto/hmac/hmac_tests.txt"}, 85 {"crypto/evp/pbkdf_test"}, 86 {"crypto/hkdf/hkdf_test"}, 87 {"crypto/hmac/hmac_test", "crypto/hmac/hmac_tests.txt"}, 88 {"crypto/lhash/lhash_test"}, 89 {"crypto/modes/gcm_test"}, 90 {"crypto/pkcs8/pkcs12_test"}, 91 {"crypto/refcount_test"}, 92 {"crypto/rsa/rsa_test"}, 93 {"crypto/thread_test"}, 94 {"crypto/x509/pkcs7_test"}, 95 {"crypto/x509v3/tab_test"}, 96 {"crypto/x509v3/v3name_test"}, 97 {"ssl/pqueue/pqueue_test"}, 98 {"ssl/ssl_test"}, 99} 100 101// testOutput is a representation of Chromium's JSON test result format. See 102// https://www.chromium.org/developers/the-json-test-results-format 103type testOutput struct { 104 Version int `json:"version"` 105 Interrupted bool `json:"interrupted"` 106 PathDelimiter string `json:"path_delimiter"` 107 SecondsSinceEpoch float64 `json:"seconds_since_epoch"` 108 NumFailuresByType map[string]int `json:"num_failures_by_type"` 109 Tests map[string]testResult `json:"tests"` 110} 111 112type testResult struct { 113 Actual string `json:"actual"` 114 Expected string `json:"expected"` 115 IsUnexpected bool `json:"is_unexpected"` 116} 117 118func newTestOutput() *testOutput { 119 return &testOutput{ 120 Version: 3, 121 PathDelimiter: ".", 122 SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond), 123 NumFailuresByType: make(map[string]int), 124 Tests: make(map[string]testResult), 125 } 126} 127 128func (t *testOutput) addResult(name, result string) { 129 if _, found := t.Tests[name]; found { 130 panic(name) 131 } 132 t.Tests[name] = testResult{ 133 Actual: result, 134 Expected: "PASS", 135 IsUnexpected: result != "PASS", 136 } 137 t.NumFailuresByType[result]++ 138} 139 140func (t *testOutput) writeTo(name string) error { 141 file, err := os.Create(name) 142 if err != nil { 143 return err 144 } 145 defer file.Close() 146 out, err := json.MarshalIndent(t, "", " ") 147 if err != nil { 148 return err 149 } 150 _, err = file.Write(out) 151 return err 152} 153 154func valgrindOf(dbAttach bool, path string, args ...string) *exec.Cmd { 155 valgrindArgs := []string{"--error-exitcode=99", "--track-origins=yes", "--leak-check=full"} 156 if dbAttach { 157 valgrindArgs = append(valgrindArgs, "--db-attach=yes", "--db-command=xterm -e gdb -nw %f %p") 158 } 159 valgrindArgs = append(valgrindArgs, path) 160 valgrindArgs = append(valgrindArgs, args...) 161 162 return exec.Command("valgrind", valgrindArgs...) 163} 164 165func gdbOf(path string, args ...string) *exec.Cmd { 166 xtermArgs := []string{"-e", "gdb", "--args"} 167 xtermArgs = append(xtermArgs, path) 168 xtermArgs = append(xtermArgs, args...) 169 170 return exec.Command("xterm", xtermArgs...) 171} 172 173type moreMallocsError struct{} 174 175func (moreMallocsError) Error() string { 176 return "child process did not exhaust all allocation calls" 177} 178 179var errMoreMallocs = moreMallocsError{} 180 181func runTestOnce(test test, mallocNumToFail int64) (passed bool, err error) { 182 prog := path.Join(*buildDir, test[0]) 183 args := test[1:] 184 var cmd *exec.Cmd 185 if *useValgrind { 186 cmd = valgrindOf(false, prog, args...) 187 } else if *useGDB { 188 cmd = gdbOf(prog, args...) 189 } else { 190 cmd = exec.Command(prog, args...) 191 } 192 var stdoutBuf bytes.Buffer 193 var stderrBuf bytes.Buffer 194 cmd.Stdout = &stdoutBuf 195 cmd.Stderr = &stderrBuf 196 if mallocNumToFail >= 0 { 197 cmd.Env = os.Environ() 198 cmd.Env = append(cmd.Env, "MALLOC_NUMBER_TO_FAIL="+strconv.FormatInt(mallocNumToFail, 10)) 199 if *mallocTestDebug { 200 cmd.Env = append(cmd.Env, "MALLOC_ABORT_ON_FAIL=1") 201 } 202 cmd.Env = append(cmd.Env, "_MALLOC_CHECK=1") 203 } 204 205 if err := cmd.Start(); err != nil { 206 return false, err 207 } 208 if err := cmd.Wait(); err != nil { 209 if exitError, ok := err.(*exec.ExitError); ok { 210 if exitError.Sys().(syscall.WaitStatus).ExitStatus() == 88 { 211 return false, errMoreMallocs 212 } 213 } 214 fmt.Print(string(stderrBuf.Bytes())) 215 return false, err 216 } 217 fmt.Print(string(stderrBuf.Bytes())) 218 219 // Account for Windows line-endings. 220 stdout := bytes.Replace(stdoutBuf.Bytes(), []byte("\r\n"), []byte("\n"), -1) 221 222 if bytes.HasSuffix(stdout, []byte("PASS\n")) && 223 (len(stdout) == 5 || stdout[len(stdout)-6] == '\n') { 224 return true, nil 225 } 226 return false, nil 227} 228 229func runTest(test test) (bool, error) { 230 if *mallocTest < 0 { 231 return runTestOnce(test, -1) 232 } 233 234 for mallocNumToFail := int64(*mallocTest); ; mallocNumToFail++ { 235 if passed, err := runTestOnce(test, mallocNumToFail); err != errMoreMallocs { 236 if err != nil { 237 err = fmt.Errorf("at malloc %d: %s", mallocNumToFail, err) 238 } 239 return passed, err 240 } 241 } 242} 243 244// shortTestName returns the short name of a test. Except for evp_test, it 245// assumes that any argument which ends in .txt is a path to a data file and not 246// relevant to the test's uniqueness. 247func shortTestName(test test) string { 248 var args []string 249 for _, arg := range test { 250 if test[0] == "crypto/evp/evp_test" || !strings.HasSuffix(arg, ".txt") { 251 args = append(args, arg) 252 } 253 } 254 return strings.Join(args, " ") 255} 256 257func main() { 258 flag.Parse() 259 260 testOutput := newTestOutput() 261 var failed []test 262 for _, test := range tests { 263 fmt.Printf("%s\n", strings.Join([]string(test), " ")) 264 265 name := shortTestName(test) 266 passed, err := runTest(test) 267 if err != nil { 268 fmt.Printf("%s failed to complete: %s\n", test[0], err) 269 failed = append(failed, test) 270 testOutput.addResult(name, "CRASHED") 271 } else if !passed { 272 fmt.Printf("%s failed to print PASS on the last line.\n", test[0]) 273 failed = append(failed, test) 274 testOutput.addResult(name, "FAIL") 275 } else { 276 testOutput.addResult(name, "PASS") 277 } 278 } 279 280 if *jsonOutput != "" { 281 if err := testOutput.writeTo(*jsonOutput); err != nil { 282 fmt.Fprintf(os.Stderr, "Error: %s\n", err) 283 } 284 } 285 286 if len(failed) > 0 { 287 fmt.Printf("\n%d of %d tests failed:\n", len(failed), len(tests)) 288 for _, test := range failed { 289 fmt.Printf("\t%s\n", strings.Join([]string(test), " ")) 290 } 291 os.Exit(1) 292 } 293 294 fmt.Printf("\nAll tests passed!\n") 295} 296