upload_system_symbols.go revision c539630b1d20f64ded4fdf76a6b7a5bc9006ce80
1/* Copyright 2014, Google Inc. 2All rights reserved. 3 4Redistribution and use in source and binary forms, with or without 5modification, are permitted provided that the following conditions are 6met: 7 8 * Redistributions of source code must retain the above copyright 9notice, this list of conditions and the following disclaimer. 10 * Redistributions in binary form must reproduce the above 11copyright notice, this list of conditions and the following disclaimer 12in the documentation and/or other materials provided with the 13distribution. 14 * Neither the name of Google Inc. nor the names of its 15contributors may be used to endorse or promote products derived from 16this software without specific prior written permission. 17 18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29*/ 30 31/* 32Tool upload_system_symbols generates and uploads Breakpad symbol files for OS X system libraries. 33 34This tool shells out to the dump_syms and symupload Breakpad tools. In its default mode, this 35will find all dynamic libraries on the system, run dump_syms to create the Breakpad symbol files, 36and then upload them to Google's crash infrastructure. 37 38The tool can also be used to only dump libraries or upload from a directory. See -help for more 39information. 40 41Both i386 and x86_64 architectures will be dumped and uploaded. 42*/ 43package main 44 45import ( 46 "flag" 47 "fmt" 48 "io" 49 "io/ioutil" 50 "log" 51 "os" 52 "os/exec" 53 "path" 54 "regexp" 55 "strings" 56 "sync" 57 "time" 58) 59 60var ( 61 breakpadTools = flag.String("breakpad-tools", "out/Release/", "Path to the Breakpad tools directory, containing dump_syms and symupload.") 62 uploadOnlyPath = flag.String("upload-from", "", "Upload a directory of symbol files that has been dumped independently.") 63 dumpOnlyPath = flag.String("dump-to", "", "Dump the symbols to the specified directory, but do not upload them.") 64 systemRoot = flag.String("system-root", "", "Path to the root of the Mac OS X system whose symbols will be dumped.") 65) 66 67var ( 68 // pathsToScan are the subpaths in the systemRoot that should be scanned for shared libraries. 69 pathsToScan = []string{ 70 "/Library/QuickTime", 71 "/System/Library/Components", 72 "/System/Library/Frameworks", 73 "/System/Library/PrivateFrameworks", 74 "/usr/lib", 75 } 76 77 // uploadServers are the list of servers to which symbols should be uploaded. 78 uploadServers = []string{ 79 "https://clients2.google.com/cr/symbol", 80 "https://clients2.google.com/cr/staging_symbol", 81 } 82 83 // blacklistRegexps match paths that should be excluded from dumping. 84 blacklistRegexps = []*regexp.Regexp{ 85 regexp.MustCompile(`/System/Library/Frameworks/Python\.framework/`), 86 regexp.MustCompile(`/System/Library/Frameworks/Ruby\.framework/`), 87 regexp.MustCompile(`_profile\.dylib$`), 88 regexp.MustCompile(`_debug\.dylib$`), 89 regexp.MustCompile(`\.a$`), 90 regexp.MustCompile(`\.dat$`), 91 } 92) 93 94func main() { 95 flag.Parse() 96 log.SetFlags(0) 97 98 var uq *UploadQueue 99 100 if *uploadOnlyPath != "" { 101 // -upload-from specified, so handle that case early. 102 uq = StartUploadQueue() 103 uploadFromDirectory(*uploadOnlyPath, uq) 104 uq.Wait() 105 return 106 } 107 108 if *systemRoot == "" { 109 log.Fatal("Need a -system-root to dump symbols for") 110 } 111 112 if *dumpOnlyPath != "" { 113 // -dump-to specified, so make sure that the path is a directory. 114 if fi, err := os.Stat(*dumpOnlyPath); err != nil { 115 log.Fatal("-dump-to location: %v", err) 116 } else if !fi.IsDir() { 117 log.Fatal("-dump-to location is not a directory") 118 } 119 } 120 121 dumpPath := *dumpOnlyPath 122 if *dumpOnlyPath == "" { 123 // If -dump-to was not specified, then run the upload pipeline and create 124 // a temporary dump output directory. 125 uq = StartUploadQueue() 126 127 if p, err := ioutil.TempDir("", "upload_system_symbols"); err != nil { 128 log.Fatal("Failed to create temporary directory: %v", err) 129 } else { 130 dumpPath = p 131 defer os.RemoveAll(p) 132 } 133 } 134 135 dq := StartDumpQueue(*systemRoot, dumpPath, uq) 136 dq.Wait() 137 if uq != nil { 138 uq.Wait() 139 } 140} 141 142type WorkerPool struct { 143 wg sync.WaitGroup 144} 145 146// StartWorkerPool will launch numWorkers goroutines all running workerFunc. 147// When workerFunc exits, the goroutine will terminate. 148func StartWorkerPool(numWorkers int, workerFunc func()) *WorkerPool { 149 p := new(WorkerPool) 150 for i := 0; i < numWorkers; i++ { 151 p.wg.Add(1) 152 go func() { 153 workerFunc() 154 p.wg.Done() 155 }() 156 } 157 return p 158} 159 160// Wait for all the workers in the pool to complete the workerFunc. 161func (p *WorkerPool) Wait() { 162 p.wg.Wait() 163} 164 165type UploadQueue struct { 166 *WorkerPool 167 queue chan string 168} 169 170// StartUploadQueue creates a new worker pool and queue, to which paths to 171// Breakpad symbol files may be sent for uploading. 172func StartUploadQueue() *UploadQueue { 173 uq := &UploadQueue{ 174 queue: make(chan string, 10), 175 } 176 uq.WorkerPool = StartWorkerPool(5, uq.worker) 177 return uq 178} 179 180// Upload enqueues the contents of filepath to be uploaded. 181func (uq *UploadQueue) Upload(filepath string) { 182 uq.queue <- filepath 183} 184 185// Done tells the queue that no more files need to be uploaded. This must be 186// called before WorkerPool.Wait. 187func (uq *UploadQueue) Done() { 188 close(uq.queue) 189} 190 191func (uq *UploadQueue) worker() { 192 symUpload := path.Join(*breakpadTools, "symupload") 193 194 for symfile := range uq.queue { 195 for _, server := range uploadServers { 196 for i := 0; i < 3; i++ { // Give each upload 3 attempts to succeed. 197 cmd := exec.Command(symUpload, symfile, server) 198 if output, err := cmd.Output(); err == nil { 199 // Success. No retry needed. 200 fmt.Printf("Uploaded %s to %s\n", symfile, server) 201 break 202 } else { 203 log.Printf("Error running symupload(%s, %s), attempt %d: %v: %s\n", symfile, server, i, err, output) 204 time.Sleep(1 * time.Second) 205 } 206 } 207 } 208 } 209} 210 211type DumpQueue struct { 212 *WorkerPool 213 dumpPath string 214 queue chan dumpRequest 215 uq *UploadQueue 216} 217 218type dumpRequest struct { 219 path string 220 arch string 221} 222 223// StartDumpQueue creates a new worker pool to find all the Mach-O libraries in 224// root and dump their symbols to dumpPath. If an UploadQueue is passed, the 225// path to the symbol file will be enqueued there, too. 226func StartDumpQueue(root, dumpPath string, uq *UploadQueue) *DumpQueue { 227 dq := &DumpQueue{ 228 dumpPath: dumpPath, 229 queue: make(chan dumpRequest), 230 uq: uq, 231 } 232 dq.WorkerPool = StartWorkerPool(12, dq.worker) 233 234 findLibsInRoot(root, dq) 235 236 return dq 237} 238 239// DumpSymbols enqueues the filepath to have its symbols dumped in the specified 240// architecture. 241func (dq *DumpQueue) DumpSymbols(filepath string, arch string) { 242 dq.queue <- dumpRequest{ 243 path: filepath, 244 arch: arch, 245 } 246} 247 248func (dq *DumpQueue) Wait() { 249 dq.WorkerPool.Wait() 250 if dq.uq != nil { 251 dq.uq.Done() 252 } 253} 254 255func (dq *DumpQueue) done() { 256 close(dq.queue) 257} 258 259func (dq *DumpQueue) worker() { 260 dumpSyms := path.Join(*breakpadTools, "dump_syms") 261 262 for req := range dq.queue { 263 filebase := path.Join(dq.dumpPath, strings.Replace(req.path, "/", "_", -1)) 264 symfile := fmt.Sprintf("%s_%s.sym", filebase, req.arch) 265 f, err := os.Create(symfile) 266 if err != nil { 267 log.Fatal("Error creating symbol file:", err) 268 } 269 270 cmd := exec.Command(dumpSyms, "-a", req.arch, req.path) 271 cmd.Stdout = f 272 err = cmd.Run() 273 f.Close() 274 275 if err != nil { 276 os.Remove(symfile) 277 log.Printf("Error running dump_syms(%s, %s): %v\n", req.arch, req.path, err) 278 } else if dq.uq != nil { 279 dq.uq.Upload(symfile) 280 } 281 } 282} 283 284// uploadFromDirectory handles the upload-only case and merely uploads all files in 285// a directory. 286func uploadFromDirectory(directory string, uq *UploadQueue) { 287 d, err := os.Open(directory) 288 if err != nil { 289 log.Fatal("Could not open directory to upload: %v", err) 290 } 291 defer d.Close() 292 293 entries, err := d.Readdirnames(0) 294 if err != nil { 295 log.Fatal("Could not read directory: %v", err) 296 } 297 298 for _, entry := range entries { 299 uq.Upload(path.Join(directory, entry)) 300 } 301 302 uq.Done() 303} 304 305// findQueue is an implementation detail of the DumpQueue that finds all the 306// Mach-O files and their architectures. 307type findQueue struct { 308 *WorkerPool 309 queue chan string 310 dq *DumpQueue 311} 312 313// findLibsInRoot looks in all the pathsToScan in the root and manages the 314// interaction between findQueue and DumpQueue. 315func findLibsInRoot(root string, dq *DumpQueue) { 316 fq := &findQueue{ 317 queue: make(chan string, 10), 318 dq: dq, 319 } 320 fq.WorkerPool = StartWorkerPool(12, fq.worker) 321 322 for _, p := range pathsToScan { 323 fq.findLibsInPath(path.Join(root, p)) 324 } 325 326 close(fq.queue) 327 fq.Wait() 328 dq.done() 329} 330 331// findLibsInPath recursively walks the directory tree, sending file paths to 332// test for being Mach-O to the findQueue. 333func (fq *findQueue) findLibsInPath(loc string) { 334 d, err := os.Open(loc) 335 if err != nil { 336 log.Fatal("Could not open %s: %v", loc, err) 337 } 338 defer d.Close() 339 340 for { 341 fis, err := d.Readdir(100) 342 if err != nil && err != io.EOF { 343 log.Fatal("Error reading directory %s: %v", loc, err) 344 } 345 346 for _, fi := range fis { 347 fp := path.Join(loc, fi.Name()) 348 if fi.IsDir() { 349 fq.findLibsInPath(fp) 350 continue 351 } else if fi.Mode()&os.ModeSymlink != 0 { 352 continue 353 } 354 355 // Test the blacklist in the worker to not slow down this main loop. 356 357 fq.queue <- fp 358 } 359 360 if err == io.EOF { 361 break 362 } 363 } 364} 365 366func (fq *findQueue) worker() { 367 for fp := range fq.queue { 368 blacklisted := false 369 for _, re := range blacklistRegexps { 370 blacklisted = blacklisted || re.MatchString(fp) 371 } 372 if blacklisted { 373 continue 374 } 375 376 imageinfos, err := GetMachOImageInfo(fp) 377 if err != nil && err != ErrNotMachO { 378 log.Printf("%s: %v", fp, err) 379 continue 380 } 381 382 for _, imageinfo := range imageinfos { 383 if imageinfo.Type == MachODylib || imageinfo.Type == MachOBundle { 384 385 fq.dq.DumpSymbols(fp, imageinfo.Arch) 386 } 387 } 388 } 389} 390