Create a new tool to upload Mac system library symbols.

R=andybons@chromium.org, mark@chromium.org

Review URL: https://breakpad.appspot.com/1124002

git-svn-id: http://google-breakpad.googlecode.com/svn/trunk@1278 4c0a9323-5329-0410-9bdc-e9ce6186880e

src/tools/mac/upload_system_symbols/arch_constants.h

@@ -0,0 +1,47 @@
+/* Copyright 2014, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#include <mach-o/fat.h>
+#include <mach-o/loader.h>
+
+// Go/Cgo does not support #define constants, so turn them into symbols
+// that are reachable from Go.
+
+const cpu_type_t kCPUType_i386 = CPU_TYPE_I386;
+const cpu_type_t kCPUType_x86_64 = CPU_TYPE_X86_64;
+
+const uint32_t kMachHeaderMagic32 = MH_MAGIC;
+const uint32_t kMachHeaderMagic64 = MH_MAGIC_64;
+const uint32_t kMachHeaderMagicFat = FAT_MAGIC;
+const uint32_t kMachHeaderCigamFat = FAT_CIGAM;
+
+const uint32_t kMachHeaderFtypeDylib = MH_DYLIB;
+const uint32_t kMachHeaderFtypeBundle = MH_BUNDLE;
+const uint32_t kMachHeaderFtypeExe = MH_EXECUTE;

src/tools/mac/upload_system_symbols/arch_reader.go

@@ -0,0 +1,268 @@
+/* Copyright 2014, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+package main
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"os"
+	"reflect"
+	"unsafe"
+)
+
+/*
+#include <mach-o/fat.h>
+#include <mach-o/loader.h>
+#include <string.h>
+
+#include "arch_constants.h"
+*/
+import "C"
+
+var (
+	ErrNotMachO        = errors.New("GetMachOImageInfo: file is not a supported Mach-O image")
+	ErrUnsupportedArch = errors.New("GetMachOImageInfo: unknown architecture detected")
+)
+
+const (
+	ArchI386   = "i386"
+	ArchX86_64 = "x86_64"
+)
+
+type MachOType int
+
+const (
+	MachODylib  MachOType = C.kMachHeaderFtypeDylib
+	MachOBundle           = C.kMachHeaderFtypeBundle
+	MachOExe              = C.kMachHeaderFtypeExe
+)
+
+type ImageInfo struct {
+	Type MachOType
+	Arch string
+}
+
+// GetMachOImageInfo will read the file at filepath and determine if it is
+// Mach-O. If it is, it will return a slice of ImageInfo that describe the
+// images in the file (may be more than one if it is a fat image).
+func GetMachOImageInfo(filepath string) ([]ImageInfo, error) {
+	f, err := os.Open(filepath)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	// Read the magic number to determine the type of file this is.
+	var magic uint32
+	err = binary.Read(f, binary.LittleEndian, &magic)
+	if err != nil {
+		return nil, err
+	}
+
+	// Rewind the file since the magic number is a field in the header
+	// structs.
+	f.Seek(0, os.SEEK_SET)
+
+	switch magic {
+	case C.kMachHeaderMagic32:
+		return readThinHeader(f, C.kMachHeaderMagic32)
+	case C.kMachHeaderMagic64:
+		return readThinHeader(f, C.kMachHeaderMagic64)
+	case C.kMachHeaderCigamFat: // Fat header is big-endian but was read in little.
+		return readFatHeader(f)
+	}
+
+	return nil, ErrNotMachO
+}
+
+func readThinHeader(f *os.File, expectedMagic uint32) ([]ImageInfo, error) {
+	var (
+		magic, filetype uint32
+		cpu             C.cpu_type_t
+		err             error
+	)
+
+	if expectedMagic == C.kMachHeaderMagic32 {
+		magic, cpu, filetype, err = readThin32Header(f)
+	} else if expectedMagic == C.kMachHeaderMagic64 {
+		magic, cpu, filetype, err = readThin64Header(f)
+	} else {
+		panic(fmt.Sprintf("Unexpected magic %#x", magic))
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	if magic != expectedMagic {
+		return nil, fmt.Errorf("readThinHeader: unexpected magic number %#x", magic)
+	}
+
+	arch := cpuTypeToArch(cpu)
+	if arch == "" {
+		return nil, ErrUnsupportedArch
+	}
+	return []ImageInfo{{MachOType(filetype), arch}}, nil
+}
+
+func readThin32Header(f *os.File) (uint32, C.cpu_type_t, uint32, error) {
+	var machHeader C.struct_mach_header
+	err := readStruct(f, binary.LittleEndian, unsafe.Pointer(&machHeader), C.struct_mach_header{})
+	if err != nil {
+		return 0, 0, 0, err
+	}
+	return uint32(machHeader.magic), machHeader.cputype, uint32(machHeader.filetype), nil
+}
+
+func readThin64Header(f *os.File) (uint32, C.cpu_type_t, uint32, error) {
+	var machHeader C.struct_mach_header_64
+	err := readStruct(f, binary.LittleEndian, unsafe.Pointer(&machHeader), C.struct_mach_header_64{})
+	if err != nil {
+		return 0, 0, 0, err
+	}
+	return uint32(machHeader.magic), machHeader.cputype, uint32(machHeader.filetype), nil
+}
+
+func readFatHeader(f *os.File) ([]ImageInfo, error) {
+	var fatHeader C.struct_fat_header
+	err := readStruct(f, binary.BigEndian, unsafe.Pointer(&fatHeader), C.struct_fat_header{})
+	if err != nil {
+		return nil, err
+	}
+
+	if fatHeader.magic != C.kMachHeaderMagicFat {
+		return nil, fmt.Errorf("readFatHeader: unexpected magic number %#x", fatHeader.magic)
+	}
+
+	// Read the fat_arch headers.
+	headers := make([]C.struct_fat_arch, fatHeader.nfat_arch)
+	for i := 0; i < int(fatHeader.nfat_arch); i++ {
+		var fatArch C.struct_fat_arch
+		err = readStruct(f, binary.BigEndian, unsafe.Pointer(&fatArch), C.struct_fat_arch{})
+		if err != nil {
+			return nil, fmt.Errorf("readFatHeader: %v", err)
+		}
+		headers[i] = fatArch
+	}
+
+	seenArches := make(map[string]int)
+
+	// Now go to each arch in the fat image and read its mach header.
+	infos := make([]ImageInfo, 0, len(headers))
+	for _, header := range headers {
+		f.Seek(int64(header.offset), os.SEEK_SET)
+
+		var thinarch []ImageInfo
+		var expectedArch string
+		switch header.cputype {
+		case C.kCPUType_i386:
+			thinarch, err = readThinHeader(f, C.kMachHeaderMagic32)
+			expectedArch = ArchI386
+		case C.kCPUType_x86_64:
+			thinarch, err = readThinHeader(f, C.kMachHeaderMagic64)
+			expectedArch = ArchX86_64
+		default:
+			err = ErrUnsupportedArch
+		}
+
+		if err != nil {
+			return nil, err
+		}
+		if thinarch[0].Arch != expectedArch {
+			return nil, fmt.Errorf("readFatHeader: expected arch %d, got %d", thinarch[0].Arch, expectedArch)
+		}
+
+		infos = append(infos, thinarch[0])
+		seenArches[thinarch[0].Arch]++
+	}
+
+	for arch, count := range seenArches {
+		if count != 1 {
+			return nil, fmt.Errorf("readFatHeader: duplicate arch %s detected", arch)
+		}
+	}
+
+	return infos, nil
+}
+
+// TODO(rsesek): Support more arches.
+func cpuTypeToArch(cpu C.cpu_type_t) string {
+	switch cpu {
+	case C.kCPUType_i386:
+		return ArchI386
+	case C.kCPUType_x86_64:
+		return ArchX86_64
+	default:
+		return ""
+	}
+}
+
+// readStruct is a incomplete version of binary.Read that uses unsafe pointers
+// to set values in unexported fields. From |f|, this will read the fields of
+// the |destType| template instance, in the specified byte |order|, and place
+// the resulting memory into |dest|.
+func readStruct(f *os.File, order binary.ByteOrder, dest unsafe.Pointer, destType interface{}) error {
+	rv := reflect.ValueOf(destType)
+	rt := rv.Type()
+	destPtr := uintptr(dest)
+
+	for i := 0; i < rv.NumField(); i++ {
+		field := rv.Field(i)
+		fieldType := rt.Field(i)
+
+		var vp unsafe.Pointer
+		var err error
+
+		switch field.Kind() {
+		case reflect.Int32:
+			var v int32
+			vp = unsafe.Pointer(&v)
+			err = binary.Read(f, order, &v)
+		case reflect.Uint32:
+			var v uint32
+			vp = unsafe.Pointer(&v)
+			err = binary.Read(f, order, &v)
+		default:
+			err = fmt.Errorf("readStruct: unsupported type %v", fieldType)
+		}
+
+		if err != nil {
+			return err
+		}
+
+		memcpy(destPtr+fieldType.Offset, vp, fieldType.Type.Size())
+	}
+	return nil
+}
+
+func memcpy(dest uintptr, value unsafe.Pointer, size uintptr) {
+	C.memcpy(unsafe.Pointer(dest), value, C.size_t(size))
+}

src/tools/mac/upload_system_symbols/arch_reader_test.go

@@ -0,0 +1,82 @@
+/* Copyright 2014, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+package main
+
+import (
+	"testing"
+)
+
+func TestFat(t *testing.T) {
+	tests := []struct {
+		file   string
+		mht    MachOType
+		arches []string
+	}{
+		{"testdata/libarchtest32.dylib", MachODylib, []string{ArchI386}},
+		{"testdata/libarchtest64.dylib", MachODylib, []string{ArchX86_64}},
+		{"testdata/libarchtest.dylib", MachODylib, []string{ArchI386, ArchX86_64}},
+		{"testdata/archtest32.exe", MachOExe, []string{ArchI386}},
+		{"testdata/archtest64.exe", MachOExe, []string{ArchX86_64}},
+		{"testdata/archtest.exe", MachOExe, []string{ArchI386, ArchX86_64}},
+	}
+
+	for _, e := range tests {
+		imageinfo, err := GetMachOImageInfo(e.file)
+		if err != nil {
+			t.Errorf("Unexpected error: %v", err)
+		}
+
+		expected := make(map[string]bool)
+		for _, arch := range e.arches {
+			expected[arch] = false
+		}
+
+		if len(imageinfo) != len(e.arches) {
+			t.Errorf("Wrong number of arches, got %d, expected %d", len(imageinfo), len(e.arches))
+		}
+
+		for _, ii := range imageinfo {
+			if ii.Type != e.mht {
+				t.Errorf("Wrong MachOType got %d, expected %d", ii.Type, e.mht)
+			}
+			if o, k := expected[ii.Arch]; o || !k {
+				t.Errorf("Unexpected architecture %q", ii.Arch)
+			}
+			expected[ii.Arch] = true
+		}
+
+		for k, v := range expected {
+			if !v {
+				t.Errorf("Did not get expected architecture %s", k)
+			}
+		}
+	}
+}

src/tools/mac/upload_system_symbols/testdata/Makefile

@@ -0,0 +1,22 @@
+all: libarchtest.dylib archtest.exe
+
+archtest32.exe: archtest.c
+	clang -m32 $< -o $@
+
+archtest64.exe: archtest.c
+	clang -m64 $< -o $@
+
+archtest.exe: archtest32.exe archtest64.exe
+	lipo $^ -create -output $@
+
+libarchtest32.dylib: archtest.c
+	clang -m32 -dynamiclib $< -o $@
+
+libarchtest64.dylib: archtest.c
+	clang -m64 -dynamiclib $< -o $@
+
+libarchtest.dylib: libarchtest32.dylib libarchtest64.dylib
+	lipo $^ -create -output $@
+
+clean:
+	rm -f *.dylib *.exe

src/tools/mac/upload_system_symbols/testdata/archtest.c

@@ -0,0 +1,7 @@
+int TestLibUsefulFunction() {
+  return 42;
+}
+
+int main() {
+  return 0;
+}

src/tools/mac/upload_system_symbols/upload_system_symbols.go

@@ -0,0 +1,389 @@
+/* Copyright 2014, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/*
+Tool upload_system_symbols generates and uploads Breakpad symbol files for OS X system libraries.
+
+This tool shells out to the dump_syms and symupload Breakpad tools. In its default mode, this
+will find all dynamic libraries on the system, run dump_syms to create the Breakpad symbol files,
+and then upload them to Google's crash infrastructure.
+
+The tool can also be used to only dump libraries or upload from a directory. See -help for more
+information.
+
+Both i386 and x86_64 architectures will be dumped and uploaded.
+*/
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+)
+
+var (
+	breakpadTools  = flag.String("breakpad-tools", "out/Release/", "Path to the Breakpad tools directory, containing dump_syms and symupload.")
+	uploadOnlyPath = flag.String("upload-from", "", "Upload a directory of symbol files that has been dumped independently.")
+	dumpOnlyPath   = flag.String("dump-to", "", "Dump the symbols to the specified directory, but do not upload them.")
+	systemRoot     = flag.String("system-root", "", "Path to the root of the Mac OS X system whose symbols will be dumped.")
+)
+
+var (
+	// pathsToScan are the subpaths in the systemRoot that should be scanned for shared libraries.
+	pathsToScan = []string{
+		"/Library/QuickTime",
+		"/System/Library/Components",
+		"/System/Library/Frameworks",
+		"/System/Library/PrivateFrameworks",
+		"/usr/lib",
+	}
+
+	// uploadServers are the list of servers to which symbols should be uploaded.
+	uploadServers = []string{
+		"https://clients2.google.com/cr/symbol",
+		"https://clients2.google.com/cr/staging_symbol",
+	}
+
+	// blacklistRegexps match paths that should be excluded from dumping.
+	blacklistRegexps = []*regexp.Regexp{
+		regexp.MustCompile(`/System/Library/Frameworks/Python\.framework/`),
+		regexp.MustCompile(`/System/Library/Frameworks/Ruby\.framework/`),
+		regexp.MustCompile(`_profile\.dylib$`),
+		regexp.MustCompile(`_debug\.dylib$`),
+		regexp.MustCompile(`\.a$`),
+		regexp.MustCompile(`\.dat$`),
+	}
+)
+
+func main() {
+	flag.Parse()
+	log.SetFlags(0)
+
+	var uq *UploadQueue
+
+	if *uploadOnlyPath != "" {
+		// -upload-from specified, so handle that case early.
+		uq = StartUploadQueue()
+		uploadFromDirectory(*uploadOnlyPath, uq)
+		uq.Wait()
+		return
+	}
+
+	if *systemRoot == "" {
+		log.Fatal("Need a -system-root to dump symbols for")
+	}
+
+	if *dumpOnlyPath != "" {
+		// -dump-to specified, so make sure that the path is a directory.
+		if fi, err := os.Stat(*dumpOnlyPath); err != nil {
+			log.Fatal("-dump-to location: %v", err)
+		} else if !fi.IsDir() {
+			log.Fatal("-dump-to location is not a directory")
+		}
+	}
+
+	dumpPath := *dumpOnlyPath
+	if *dumpOnlyPath == "" {
+		// If -dump-to was not specified, then run the upload pipeline and create
+		// a temporary dump output directory.
+		uq = StartUploadQueue()
+
+		if p, err := ioutil.TempDir("", "upload_system_symbols"); err != nil {
+			log.Fatal("Failed to create temporary directory: %v", err)
+		} else {
+			dumpPath = p
+			defer os.RemoveAll(p)
+		}
+	}
+
+	dq := StartDumpQueue(*systemRoot, dumpPath, uq)
+	dq.Wait()
+	if uq != nil {
+		uq.Wait()
+	}
+}
+
+type WorkerPool struct {
+	wg sync.WaitGroup
+}
+
+// StartWorkerPool will launch numWorkers goroutines all running workerFunc.
+// When workerFunc exits, the goroutine will terminate.
+func StartWorkerPool(numWorkers int, workerFunc func()) *WorkerPool {
+	p := new(WorkerPool)
+	for i := 0; i < numWorkers; i++ {
+		p.wg.Add(1)
+		go func() {
+			workerFunc()
+			p.wg.Done()
+		}()
+	}
+	return p
+}
+
+// Wait for all the workers in the pool to complete the workerFunc.
+func (p *WorkerPool) Wait() {
+	p.wg.Wait()
+}
+
+type UploadQueue struct {
+	*WorkerPool
+	queue chan string
+}
+
+// StartUploadQueue creates a new worker pool and queue, to which paths to
+// Breakpad symbol files may be sent for uploading.
+func StartUploadQueue() *UploadQueue {
+	uq := &UploadQueue{
+		queue: make(chan string, 10),
+	}
+	uq.WorkerPool = StartWorkerPool(5, uq.worker)
+	return uq
+}
+
+// Upload enqueues the contents of filepath to be uploaded.
+func (uq *UploadQueue) Upload(filepath string) {
+	uq.queue <- filepath
+}
+
+// Done tells the queue that no more files need to be uploaded. This must be
+// called before WorkerPool.Wait.
+func (uq *UploadQueue) Done() {
+	close(uq.queue)
+}
+
+func (uq *UploadQueue) worker() {
+	symUpload := path.Join(*breakpadTools, "symupload")
+
+	for symfile := range uq.queue {
+		for _, server := range uploadServers {
+			for i := 0; i < 3; i++ { // Give each upload 3 attempts to succeed.
+				cmd := exec.Command(symUpload, symfile, server)
+				if output, err := cmd.Output(); err == nil {
+					// Success. No retry needed.
+					fmt.Printf("Uploaded %s to %s\n", symfile, server)
+					break
+				} else {
+					log.Printf("Error running symupload(%s, %s), attempt %d: %v: %s\n", symfile, server, i, err, output)
+					time.Sleep(1 * time.Second)
+				}
+			}
+		}
+	}
+}
+
+type DumpQueue struct {
+	*WorkerPool
+	dumpPath string
+	queue    chan dumpRequest
+	uq       *UploadQueue
+}
+
+type dumpRequest struct {
+	path string
+	arch string
+}
+
+// StartDumpQueue creates a new worker pool to find all the Mach-O libraries in
+// root and dump their symbols to dumpPath. If an UploadQueue is passed, the
+// path to the symbol file will be enqueued there, too.
+func StartDumpQueue(root, dumpPath string, uq *UploadQueue) *DumpQueue {
+	dq := &DumpQueue{
+		dumpPath: dumpPath,
+		queue:    make(chan dumpRequest),
+		uq:       uq,
+	}
+	dq.WorkerPool = StartWorkerPool(12, dq.worker)
+
+	findLibsInRoot(root, dq)
+
+	return dq
+}
+
+// DumpSymbols enqueues the filepath to have its symbols dumped in the specified
+// architecture.
+func (dq *DumpQueue) DumpSymbols(filepath string, arch string) {
+	dq.queue <- dumpRequest{
+		path: filepath,
+		arch: arch,
+	}
+}
+
+func (dq *DumpQueue) Wait() {
+	dq.WorkerPool.Wait()
+	if dq.uq != nil {
+		dq.uq.Done()
+	}
+}
+
+func (dq *DumpQueue) done() {
+	close(dq.queue)
+}
+
+func (dq *DumpQueue) worker() {
+	dumpSyms := path.Join(*breakpadTools, "dump_syms")
+
+	for req := range dq.queue {
+		filebase := path.Join(dq.dumpPath, strings.Replace(req.path, "/", "_", -1))
+		symfile := fmt.Sprintf("%s_%s.sym", filebase, req.arch)
+		f, err := os.Create(symfile)
+		if err != nil {
+			log.Fatal("Error creating symbol file:", err)
+		}
+
+		cmd := exec.Command(dumpSyms, "-a", req.arch, req.path)
+		cmd.Stdout = f
+		err = cmd.Run()
+		f.Close()
+
+		if err != nil {
+			os.Remove(symfile)
+			log.Printf("Error running dump_syms(%s, %s): %v\n", req.arch, req.path, err)
+		} else if dq.uq != nil {
+			dq.uq.Upload(symfile)
+		}
+	}
+}
+
+// uploadFromDirectory handles the upload-only case and merely uploads all files in
+// a directory.
+func uploadFromDirectory(directory string, uq *UploadQueue) {
+	d, err := os.Open(directory)
+	if err != nil {
+		log.Fatal("Could not open directory to upload: %v", err)
+	}
+	defer d.Close()
+
+	entries, err := d.Readdirnames(0)
+	if err != nil {
+		log.Fatal("Could not read directory: %v", err)
+	}
+
+	for _, entry := range entries {
+		uq.Upload(path.Join(directory, entry))
+	}
+
+	uq.Done()
+}
+
+// findQueue is an implementation detail of the DumpQueue that finds all the
+// Mach-O files and their architectures.
+type findQueue struct {
+	*WorkerPool
+	queue chan string
+	dq    *DumpQueue
+}
+
+// findLibsInRoot looks in all the pathsToScan in the root and manages the
+// interaction between findQueue and DumpQueue.
+func findLibsInRoot(root string, dq *DumpQueue) {
+	fq := &findQueue{
+		queue: make(chan string, 10),
+		dq:    dq,
+	}
+	fq.WorkerPool = StartWorkerPool(12, fq.worker)
+
+	for _, p := range pathsToScan {
+		fq.findLibsInPath(path.Join(root, p))
+	}
+
+	close(fq.queue)
+	fq.Wait()
+	dq.done()
+}
+
+// findLibsInPath recursively walks the directory tree, sending file paths to
+// test for being Mach-O to the findQueue.
+func (fq *findQueue) findLibsInPath(loc string) {
+	d, err := os.Open(loc)
+	if err != nil {
+		log.Fatal("Could not open %s: %v", loc, err)
+	}
+	defer d.Close()
+
+	for {
+		fis, err := d.Readdir(100)
+		if err != nil && err != io.EOF {
+			log.Fatal("Error reading directory %s: %v", loc, err)
+		}
+
+		for _, fi := range fis {
+			fp := path.Join(loc, fi.Name())
+			if fi.IsDir() {
+				fq.findLibsInPath(fp)
+				continue
+			} else if fi.Mode()&os.ModeSymlink != 0 {
+				continue
+			}
+
+			// Test the blacklist in the worker to not slow down this main loop.
+
+			fq.queue <- fp
+		}
+
+		if err == io.EOF {
+			break
+		}
+	}
+}
+
+func (fq *findQueue) worker() {
+	for fp := range fq.queue {
+		blacklisted := false
+		for _, re := range blacklistRegexps {
+			blacklisted = blacklisted || re.MatchString(fp)
+		}
+		if blacklisted {
+			continue
+		}
+
+		imageinfos, err := GetMachOImageInfo(fp)
+		if err != nil && err != ErrNotMachO {
+			log.Printf("%s: %v", fp, err)
+			continue
+		}
+
+		for _, imageinfo := range imageinfos {
+			if imageinfo.Type == MachODylib || imageinfo.Type == MachOBundle {
+
+				fq.dq.DumpSymbols(fp, imageinfo.Arch)
+			}
+		}
+	}
+}