From 50e04befeca9ae63296a73c8d5d2870b904971b4 Mon Sep 17 00:00:00 2001
From: diamondburned <datutbrus@gmail.com>
Date: Thu, 5 Aug 2021 16:47:49 -0700
Subject: [PATCH] cmd/cgo: concurrent file generation

This commit allows cmd/cgo to generate files in parallel to each other.
The parallelism is determined by either the thread count (using
$GOMAXPROCS) or the -nparallel flag, which is set in
cmd/go/internal/work to be maximum 4.

The commit addresses some existing race conditions that had to do with
mutating the global state by using a global write mutex during the
translation stage.

Other race conditions that had to do with mutating the Package singleton
was solved by giving each translation worker its own package, and then
combining them back into a single package.

With this and the cmd/go/internal/work changes in place, build time
(from scratch, with -a) for gotktrix is reduced from 33 minutes to 10
minutes on an 8-thread Intel Core i5-8250U.
---
 src/cmd/cgo/gcc.go               | 120 +++++++++++-----
 src/cmd/cgo/godefs.go            |   6 +-
 src/cmd/cgo/main.go              | 238 ++++++++++++++++++++++++-------
 src/cmd/cgo/out.go               |  20 ++-
 src/cmd/cgo/util.go              |  13 +-
 src/cmd/go/internal/work/exec.go |   7 +-
 6 files changed, 303 insertions(+), 101 deletions(-)

diff --git a/src/cmd/cgo/gcc.go b/src/cmd/cgo/gcc.go
index 997a830994f0..59a8405c0c71 100644
--- a/src/cmd/cgo/gcc.go
+++ b/src/cmd/cgo/gcc.go
@@ -26,6 +26,8 @@ import (
 	"os/exec"
 	"strconv"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"unicode"
 	"unicode/utf8"
 
@@ -485,7 +487,7 @@ func (p *Package) guessKinds(f *File) []*Name {
 		}
 		needType = append(needType, n)
 	}
-	if nerrors > 0 {
+	if nerrors.has() {
 		// Check if compiling the preamble by itself causes any errors,
 		// because the messages we've printed out so far aren't helpful
 		// to users debugging preamble mistakes. See issue 8442.
@@ -727,9 +729,12 @@ func (p *Package) prepareNames(f *File) {
 			}
 		}
 		p.mangleName(n)
+
+		globalMu.Lock()
 		if n.Kind == "type" && typedef[n.Mangle] == nil {
 			typedef[n.Mangle] = n.Type
 		}
+		globalMu.Unlock()
 	}
 }
 
@@ -1023,7 +1028,7 @@ func (p *Package) hasPointer(f *File, t ast.Expr, top bool) bool {
 		}
 		// Check whether this is a pointer to a C union (or class)
 		// type that contains a pointer.
-		if unionWithPointer[t.X] {
+		if isUnionWithPointer(t.X) {
 			return true
 		}
 		return p.hasPointer(f, t.X, false)
@@ -1046,7 +1051,10 @@ func (p *Package) hasPointer(f *File, t ast.Expr, top bool) bool {
 				}
 			}
 		}
-		if def := typedef[t.Name]; def != nil {
+		globalMu.Lock()
+		def := typedef[t.Name]
+		globalMu.Unlock()
+		if def != nil {
 			return p.hasPointer(f, def.Go, top)
 		}
 		if t.Name == "string" {
@@ -1582,7 +1590,7 @@ func checkGCCBaseCmd() ([]string, error) {
 }
 
 // gccMachine returns the gcc -m flag to use, either "-m32", "-m64" or "-marm".
-func (p *Package) gccMachine() []string {
+func gccMachine() []string {
 	switch goarch {
 	case "amd64":
 		if goos == "darwin" {
@@ -1617,20 +1625,20 @@ func (p *Package) gccMachine() []string {
 	return nil
 }
 
-func gccTmp() string {
-	return *objDir + "_cgo_.o"
+func (p *Package) gccTmp() string {
+	return fmt.Sprintf("%s_%d_cgo_.o", *objDir, p.workerID)
 }
 
 // gccCmd returns the gcc command line to use for compiling
 // the input.
 func (p *Package) gccCmd() []string {
 	c := append(gccBaseCmd,
-		"-w",          // no warnings
-		"-Wno-error",  // warnings are not errors
-		"-o"+gccTmp(), // write object to tmp
-		"-gdwarf-2",   // generate DWARF v2 debugging symbols
-		"-c",          // do not link
-		"-xc",         // input language is C
+		"-w",            // no warnings
+		"-Wno-error",    // warnings are not errors
+		"-o"+p.gccTmp(), // write object to tmp
+		"-gdwarf-2",     // generate DWARF v2 debugging symbols
+		"-c",            // do not link
+		"-xc",           // input language is C
 	)
 	if p.GccIsClang {
 		c = append(c,
@@ -1653,7 +1661,7 @@ func (p *Package) gccCmd() []string {
 	}
 
 	c = append(c, p.GccOptions...)
-	c = append(c, p.gccMachine()...)
+	c = append(c, gccMachine()...)
 	if goos == "aix" {
 		c = append(c, "-maix64")
 		c = append(c, "-mcmodel=large")
@@ -1717,11 +1725,11 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
 		}
 	}
 
-	if f, err := macho.Open(gccTmp()); err == nil {
+	if f, err := macho.Open(p.gccTmp()); err == nil {
 		defer f.Close()
 		d, err := f.DWARF()
 		if err != nil {
-			fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
+			fatalf("cannot load DWARF output from %s: %v", p.gccTmp(), err)
 		}
 		bo := f.ByteOrder
 		if f.Symtab != nil {
@@ -1795,11 +1803,11 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
 		return d, ints, floats, strs
 	}
 
-	if f, err := elf.Open(gccTmp()); err == nil {
+	if f, err := elf.Open(p.gccTmp()); err == nil {
 		defer f.Close()
 		d, err := f.DWARF()
 		if err != nil {
-			fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
+			fatalf("cannot load DWARF output from %s: %v", p.gccTmp(), err)
 		}
 		bo := f.ByteOrder
 		symtab, err := f.Symbols()
@@ -1874,11 +1882,11 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
 		return d, ints, floats, strs
 	}
 
-	if f, err := pe.Open(gccTmp()); err == nil {
+	if f, err := pe.Open(p.gccTmp()); err == nil {
 		defer f.Close()
 		d, err := f.DWARF()
 		if err != nil {
-			fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
+			fatalf("cannot load DWARF output from %s: %v", p.gccTmp(), err)
 		}
 		bo := binary.LittleEndian
 		for _, s := range f.Symbols {
@@ -1946,11 +1954,11 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
 		return d, ints, floats, strs
 	}
 
-	if f, err := xcoff.Open(gccTmp()); err == nil {
+	if f, err := xcoff.Open(p.gccTmp()); err == nil {
 		defer f.Close()
 		d, err := f.DWARF()
 		if err != nil {
-			fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
+			fatalf("cannot load DWARF output from %s: %v", p.gccTmp(), err)
 		}
 		bo := binary.BigEndian
 		for _, s := range f.Symbols {
@@ -2016,7 +2024,7 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
 		buildStrings()
 		return d, ints, floats, strs
 	}
-	fatalf("cannot parse gcc output %s as ELF, Mach-O, PE, XCOFF object", gccTmp())
+	fatalf("cannot parse gcc output %s as ELF, Mach-O, PE, XCOFF object", p.gccTmp())
 	panic("not reached")
 }
 
@@ -2026,7 +2034,7 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
 // and its included files.
 func (p *Package) gccDefines(stdin []byte) string {
 	base := append(gccBaseCmd, "-E", "-dM", "-xc")
-	base = append(base, p.gccMachine()...)
+	base = append(base, gccMachine()...)
 	stdout, _ := runGcc(stdin, append(append(base, p.GccOptions...), "-"))
 	return stdout
 }
@@ -2126,7 +2134,13 @@ type typeConv struct {
 	intSize int64
 }
 
-var tagGen int
+var tagGenCounter int64
+
+func genTag() int {
+	return int(atomic.AddInt64(&tagGenCounter, 1) - 1)
+}
+
+var globalMu sync.Mutex
 var typedef = make(map[string]*Type)
 var goIdent = make(map[string]*ast.Ident)
 
@@ -2134,6 +2148,29 @@ var goIdent = make(map[string]*ast.Ident)
 // that may contain a pointer. This is used for cgo pointer checking.
 var unionWithPointer = make(map[ast.Expr]bool)
 
+func unionWithPointerTrueIfKey(setKey, ifKey ast.Expr) {
+	globalMu.Lock()
+	defer globalMu.Unlock()
+
+	if unionWithPointer[ifKey] {
+		unionWithPointer[setKey] = true
+	}
+}
+
+func isUnionWithPointer(key ast.Expr) bool {
+	globalMu.Lock()
+	defer globalMu.Unlock()
+
+	return unionWithPointer[key]
+}
+
+func unionWithPointerTrue(key ast.Expr) {
+	globalMu.Lock()
+	defer globalMu.Unlock()
+
+	unionWithPointer[key] = true
+}
+
 // anonymousStructTag provides a consistent tag for an anonymous struct.
 // The same dwarf.StructType pointer will always get the same tag.
 var anonymousStructTag = make(map[*dwarf.StructType]string)
@@ -2472,9 +2509,7 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
 		t.Size = t1.Size
 		t.Align = t1.Align
 		t.Go = t1.Go
-		if unionWithPointer[t1.Go] {
-			unionWithPointer[t.Go] = true
-		}
+		unionWithPointerTrueIfKey(t.Go, t1.Go)
 		t.EnumValues = nil
 		t.Typedef = ""
 		t.C.Set("%s "+dt.Qual, t1.C)
@@ -2488,18 +2523,21 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
 			break
 		}
 		if tag == "" {
+			globalMu.Lock()
 			tag = anonymousStructTag[dt]
 			if tag == "" {
-				tag = "__" + strconv.Itoa(tagGen)
-				tagGen++
+				tag = "__" + strconv.Itoa(genTag())
 				anonymousStructTag[dt] = tag
 			}
+			globalMu.Unlock()
 		} else if t.C.Empty() {
 			t.C.Set(dt.Kind + " " + tag)
 		}
 		name := c.Ident("_Ctype_" + dt.Kind + "_" + tag)
 		t.Go = name // publish before recursive calls
+		globalMu.Lock()
 		goIdent[name.Name] = name
+		globalMu.Unlock()
 		if dt.ByteSize < 0 {
 			// Size calculation in c.Struct/c.Opaque will die with size=-1 (unknown),
 			// so execute the basic things that the struct case would do
@@ -2519,20 +2557,24 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
 				// on the Go heap, right? It currently doesn't work for unions because
 				// they are defined as a type alias for struct{}, not a defined type.
 			}
+			globalMu.Lock()
 			typedef[name.Name] = &tt
+			globalMu.Unlock()
 			break
 		}
 		switch dt.Kind {
 		case "class", "union":
 			t.Go = c.Opaque(t.Size)
 			if c.dwarfHasPointer(dt, pos) {
-				unionWithPointer[t.Go] = true
+				unionWithPointerTrue(t.Go)
 			}
 			if t.C.Empty() {
 				t.C.Set("__typeof__(unsigned char[%d])", t.Size)
 			}
 			t.Align = 1 // TODO: should probably base this on field alignment.
+			globalMu.Lock()
 			typedef[name.Name] = t
+			globalMu.Unlock()
 		case "struct":
 			g, csyntax, align := c.Struct(dt, pos)
 			if t.C.Empty() {
@@ -2545,7 +2587,9 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
 			}
 			tt.Go = g
 			tt.NotInHeap = c.notInHeapStructs[tag]
+			globalMu.Lock()
 			typedef[name.Name] = &tt
+			globalMu.Unlock()
 		}
 
 	case *dwarf.TypedefType:
@@ -2568,7 +2612,9 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
 			break
 		}
 		name := c.Ident("_Ctype_" + dt.Name)
+		globalMu.Lock()
 		goIdent[name.Name] = name
+		globalMu.Unlock()
 		akey := ""
 		if c.anonymousStructTypedef(dt) {
 			// only load type recursively for typedefs of anonymous
@@ -2583,10 +2629,12 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
 			s.BadPointer = true
 			sub = &s
 			// Make sure we update any previously computed type.
+			globalMu.Lock()
 			if oldType := typedef[name.Name]; oldType != nil {
 				oldType.Go = sub.Go
 				oldType.BadPointer = true
 			}
+			globalMu.Unlock()
 		}
 		if c.badVoidPointerTypedef(dt) {
 			// Treat this typedef as a pointer to a NotInHeap void.
@@ -2615,11 +2663,10 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
 		t.Go = name
 		t.BadPointer = sub.BadPointer
 		t.NotInHeap = sub.NotInHeap
-		if unionWithPointer[sub.Go] {
-			unionWithPointer[t.Go] = true
-		}
+		unionWithPointerTrueIfKey(t.Go, sub.Go)
 		t.Size = sub.Size
 		t.Align = sub.Align
+		globalMu.Lock()
 		oldType := typedef[name.Name]
 		if oldType == nil {
 			tt := *t
@@ -2628,6 +2675,7 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
 			tt.NotInHeap = sub.NotInHeap
 			typedef[name.Name] = &tt
 		}
+		globalMu.Unlock()
 
 		// If sub.Go.Name is "_Ctype_struct_foo" or "_Ctype_union_foo" or "_Ctype_class_foo",
 		// use that as the Go form for this typedef too, so that the typedef will be interchangeable
@@ -2638,7 +2686,9 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
 
 			if isStructUnionClass(sub.Go) {
 				// Use the typedef name for C code.
+				globalMu.Lock()
 				typedef[sub.Go.(*ast.Ident).Name].C = t.C
+				globalMu.Unlock()
 			}
 
 			// If we've seen this typedef before, and it
@@ -2698,8 +2748,12 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
 			}
 			s = strings.Replace(s, " ", "", -1)
 			name := c.Ident("_Ctype_" + s)
+
 			tt := *t
+			globalMu.Lock()
 			typedef[name.Name] = &tt
+			globalMu.Unlock()
+
 			if !*godefs {
 				t.Go = name
 			}
diff --git a/src/cmd/cgo/godefs.go b/src/cmd/cgo/godefs.go
index c0d59aee01d7..740aa3bc30b2 100644
--- a/src/cmd/cgo/godefs.go
+++ b/src/cmd/cgo/godefs.go
@@ -115,11 +115,11 @@ func (p *Package) godefs(f *File) string {
 	return buf.String()
 }
 
-var gofmtBuf bytes.Buffer
-
 // gofmt returns the gofmt-formatted string for an AST node.
 func gofmt(n interface{}) string {
-	gofmtBuf.Reset()
+	var gofmtBuf strings.Builder
+	gofmtBuf.Grow(512)
+
 	err := printer.Fprint(&gofmtBuf, fset, n)
 	if err != nil {
 		return "<" + err.Error() + ">"
diff --git a/src/cmd/cgo/main.go b/src/cmd/cgo/main.go
index 14642b7576b0..36e76908c602 100644
--- a/src/cmd/cgo/main.go
+++ b/src/cmd/cgo/main.go
@@ -26,6 +26,7 @@ import (
 	"runtime"
 	"sort"
 	"strings"
+	"sync"
 
 	"cmd/internal/edit"
 	"cmd/internal/objabi"
@@ -33,6 +34,8 @@ import (
 
 // A Package collects information about the package we're going to write.
 type Package struct {
+	workerID int
+
 	PackageName string // name of package
 	PackagePath string
 	PtrSize     int64
@@ -40,17 +43,36 @@ type Package struct {
 	GccOptions  []string
 	GccIsClang  bool
 	CgoFlags    map[string][]string // #cgo flags (CFLAGS, LDFLAGS)
-	Written     map[string]bool
-	Name        map[string]*Name // accumulated Name from Files
+	Written     *WrittenFiles
+	Name        map[string]*Name // TODO: parallelize. accumulated Name from Files
 	ExpFunc     []*ExpFunc       // accumulated ExpFunc from Files
-	Decl        []ast.Decl
-	GoFiles     []string        // list of Go files
-	GccFiles    []string        // list of gcc output files
-	Preamble    string          // collected preamble for _cgo_export.h
+	Decl        []ast.Decl       `json:"-"`
+	GoFiles     []string         // list of Go files
+	GccFiles    []string         // list of gcc output files
+	Preamble    string           // collected preamble for _cgo_export.h
+
 	typedefs    map[string]bool // type names that appear in the types of the objects we're interested in
 	typedefList []typedefInfo
 }
 
+type WrittenFiles struct {
+	mut     sync.Mutex
+	written map[string]struct{}
+}
+
+func (w *WrittenFiles) Mark(file string) (shouldWrite bool) {
+	w.mut.Lock()
+	defer w.mut.Unlock()
+
+	_, already := w.written[file]
+	if already {
+		return false
+	}
+
+	w.written[file] = struct{}{}
+	return true
+}
+
 // A typedefInfo is an element on Package.typedefList: a typedef name
 // and the position where it was required.
 type typedefInfo struct {
@@ -87,7 +109,7 @@ func nameKeys(m map[string]*Name) []string {
 
 // A Call refers to a call of a C.xxx function in the AST.
 type Call struct {
-	Call     *ast.CallExpr
+	Call     *ast.CallExpr `json:"-"`
 	Deferred bool
 	Done     bool
 }
@@ -95,7 +117,7 @@ type Call struct {
 // A Ref refers to an expression of the form C.xxx in the AST.
 type Ref struct {
 	Name    *Name
-	Expr    *ast.Expr
+	Expr    *ast.Expr `json:"-"`
 	Context astContext
 	Done    bool
 }
@@ -108,13 +130,13 @@ var nameKinds = []string{"iconst", "fconst", "sconst", "type", "var", "fpvar", "
 
 // A Name collects information about C.xxx.
 type Name struct {
-	Go       string // name used in Go referring to package C
-	Mangle   string // name used in generated Go
-	C        string // name used in C
-	Define   string // #define expansion
-	Kind     string // one of the nameKinds
-	Type     *Type  // the type of xxx
-	FuncType *FuncType
+	Go       string    // name used in Go referring to package C
+	Mangle   string    // name used in generated Go
+	C        string    // name used in C
+	Define   string    // #define expansion
+	Kind     string    // one of the nameKinds
+	Type     *Type     `json:"-"` // the type of xxx
+	FuncType *FuncType `json:"-"`
 	AddError bool
 	Const    string // constant definition
 }
@@ -133,23 +155,23 @@ func (n *Name) IsConst() bool {
 // Such functions are identified in the Go input file
 // by doc comments containing the line //export ExpName
 type ExpFunc struct {
-	Func    *ast.FuncDecl
-	ExpName string // name to use from C
+	Func    *ast.FuncDecl `json:"-"`
+	ExpName string        // name to use from C
 	Doc     string
 }
 
 // A TypeRepr contains the string representation of a type.
 type TypeRepr struct {
 	Repr       string
-	FormatArgs []interface{}
+	FormatArgs []interface{} `json:"-"`
 }
 
 // A Type collects information about a type in both the C and Go worlds.
 type Type struct {
 	Size       int64
 	Align      int64
-	C          *TypeRepr
-	Go         ast.Expr
+	C          *TypeRepr `json:"-"`
+	Go         ast.Expr  `json:"-"`
 	EnumValues map[string]int64
 	Typedef    string
 	BadPointer bool // this pointer type should be represented as a uintptr (deprecated)
@@ -223,6 +245,8 @@ var cPrefix string
 
 var fset = token.NewFileSet()
 
+var nparallel = flag.Int("nparallel", runtime.GOMAXPROCS(0), "maximum parallel jobs")
+
 var dynobj = flag.String("dynimport", "", "if non-empty, print dynamic import data for that file")
 var dynout = flag.String("dynout", "", "write -dynimport output to this file")
 var dynpackage = flag.String("dynpackage", "main", "set Go package for -dynimport output")
@@ -241,11 +265,13 @@ var exportHeader = flag.String("exportheader", "", "where to write export header
 var gccgo = flag.Bool("gccgo", false, "generate files for use with gccgo")
 var gccgoprefix = flag.String("gccgoprefix", "", "-fgo-prefix option used with gccgo")
 var gccgopkgpath = flag.String("gccgopkgpath", "", "-fgo-pkgpath option used with gccgo")
-var gccgoMangler func(string) string
 var importRuntimeCgo = flag.Bool("import_runtime_cgo", true, "import runtime/cgo in generated code")
 var importSyscall = flag.Bool("import_syscall", true, "import syscall in generated code")
 var trimpath = flag.String("trimpath", "", "applies supplied rewrites or trims prefixes to recorded source file paths")
 
+var gccgoMangler func(string) string
+var gccgoManglerOnce sync.Once
+
 var goarch, goos, gomips, gomips64 string
 var gccBaseCmd []string
 
@@ -302,7 +328,7 @@ func main() {
 		}
 	}
 
-	p := newPackage(args[:i])
+	p := newPackage(args[:i], 0)
 
 	// We need a C compiler to be available. Check this.
 	var err error
@@ -370,43 +396,152 @@ func main() {
 	}
 	*objDir += string(filepath.Separator)
 
-	for i, input := range goFiles {
-		f := fs[i]
-		p.Translate(f)
-		for _, cref := range f.Ref {
-			switch cref.Context {
-			case ctxCall, ctxCall2:
-				if cref.Name.Kind != "type" {
-					break
-				}
-				old := *cref.Expr
-				*cref.Expr = cref.Name.Type.Go
-				f.Edit.Replace(f.offset(old.Pos()), f.offset(old.End()), gofmt(cref.Name.Type.Go))
-			}
-		}
-		if nerrors > 0 {
-			os.Exit(2)
-		}
-		p.PackagePath = f.Package
-		p.Record(f)
-		if *godefs {
-			os.Stdout.WriteString(p.godefs(f))
-		} else {
-			p.writeOutput(f, input)
+	pkgs := doFiles(p, goFiles, fs)
+
+	if !*godefs {
+		pkg := coalescePackages(pkgs)
+		pkg.writeDefs()
+	}
+
+	if nerrors.get() > 0 {
+		os.Exit(2)
+	}
+}
+
+type doFileJob struct {
+	input string
+	file  *File
+}
+
+func doFiles(p *Package, inputs []string, files []*File) []*Package {
+	if len(inputs) != len(files) {
+		fatalf("unexpected len(inputs) %d != len(files) %d", len(inputs), len(files))
+	}
+
+	jobCh := make(chan doFileJob)
+
+	var wg sync.WaitGroup
+	wg.Add(*nparallel)
+
+	pkgs := make([]*Package, *nparallel)
+	pkgs[0] = p
+
+	for n := 1; n < *nparallel; n++ {
+		pkgs[n] = cloneNewPackage(p, n)
+	}
+
+	for n := 0; n < *nparallel; n++ {
+		p := pkgs[n]
+		go func() {
+			doFileWorker(p, jobCh)
+			wg.Done()
+		}()
+	}
+
+	for i := range inputs {
+		jobCh <- doFileJob{
+			input: inputs[i],
+			file:  files[i],
 		}
 	}
 
-	if !*godefs {
-		p.writeDefs()
+	close(jobCh)
+	wg.Wait()
+
+	return pkgs
+}
+
+func doFileWorker(p *Package, jobCh <-chan doFileJob) *Package {
+	for job := range jobCh {
+		doFile(p, job.input, job.file)
 	}
-	if nerrors > 0 {
+
+	return p
+}
+
+func doFile(p *Package, input string, f *File) {
+	p.Translate(f)
+	for _, cref := range f.Ref {
+		switch cref.Context {
+		case ctxCall, ctxCall2:
+			if cref.Name.Kind != "type" {
+				break
+			}
+			old := *cref.Expr
+			*cref.Expr = cref.Name.Type.Go
+			f.Edit.Replace(f.offset(old.Pos()), f.offset(old.End()), gofmt(cref.Name.Type.Go))
+		}
+	}
+	if nerrors.get() > 0 {
 		os.Exit(2)
 	}
+	p.PackagePath = f.Package
+	p.Record(f)
+	if *godefs {
+		os.Stdout.WriteString(p.godefs(f))
+	} else {
+		p.writeOutput(f, input)
+	}
+}
+
+func cloneNewPackage(p *Package, workerID int) *Package {
+	return &Package{
+		workerID:    workerID,
+		PackageName: p.PackageName,
+		PackagePath: p.PackagePath,
+		PtrSize:     p.PtrSize,
+		IntSize:     p.IntSize,
+		GccOptions:  p.GccOptions, // init in main only
+		GccIsClang:  p.GccIsClang,
+		CgoFlags:    p.CgoFlags, // init in main only
+		Written:     p.Written,
+	}
+}
+
+func coalescePackages(pkgs []*Package) *Package {
+	if len(pkgs) == 0 {
+		return nil
+	}
+
+	nonEmpty := pkgs[:0]
+	for _, p := range pkgs {
+		if p.PackageName != "" {
+			nonEmpty = append(nonEmpty, p)
+		}
+	}
+
+	pkgs = nonEmpty
+	pkg := pkgs[0]
+
+	for _, p := range pkgs[1:] {
+		pkg.PackageName = p.PackageName
+		pkg.PackagePath = p.PackagePath
+		pkg.ExpFunc = append(pkg.ExpFunc, p.ExpFunc...)
+		pkg.Decl = append(pkg.Decl, p.Decl...)
+		pkg.GoFiles = append(pkg.GoFiles, p.GoFiles...)
+		pkg.GccFiles = append(pkg.GccFiles, p.GccFiles...)
+		pkg.Preamble += p.Preamble
+
+		if pkg.Name == nil && p.Name != nil {
+			pkg.Name = p.Name
+			continue
+		}
+
+		for k, v := range p.Name {
+			if o, ok := pkg.Name[k]; ok && !reflect.DeepEqual(v, o) {
+				fatalf("duplicate pkg.Name %s, %v == %v", k, v, o)
+			}
+
+			pkg.Name[k] = v
+		}
+	}
+
+	return pkg
 }
 
 // newPackage returns a new Package that will invoke
 // gcc with the additional arguments specified in args.
-func newPackage(args []string) *Package {
+func newPackage(args []string, workerID int) *Package {
 	goarch = runtime.GOARCH
 	if s := os.Getenv("GOARCH"); s != "" {
 		goarch = s
@@ -432,10 +567,13 @@ func newPackage(args []string) *Package {
 	os.Setenv("LC_ALL", "C")
 
 	p := &Package{
+		workerID: workerID,
 		PtrSize:  ptrSize,
 		IntSize:  intSize,
 		CgoFlags: make(map[string][]string),
-		Written:  make(map[string]bool),
+		Written: &WrittenFiles{
+			written: make(map[string]struct{}),
+		},
 	}
 	p.addToFlag("CFLAGS", args)
 	return p
diff --git a/src/cmd/cgo/out.go b/src/cmd/cgo/out.go
index 4968f7059d9b..b934d12d9423 100644
--- a/src/cmd/cgo/out.go
+++ b/src/cmd/cgo/out.go
@@ -32,14 +32,13 @@ var (
 
 // writeDefs creates output files to be compiled by gc and gcc.
 func (p *Package) writeDefs() {
-	var fgo2, fc io.Writer
-	f := creat(*objDir + "_cgo_gotypes.go")
-	defer f.Close()
-	fgo2 = f
+	fgo2 := creat(*objDir + "_cgo_gotypes.go")
+	defer fgo2.Close()
+
+	var fc *os.File
 	if *gccgo {
-		f := creat(*objDir + "_cgo_defun.c")
-		defer f.Close()
-		fc = f
+		fc = creat(*objDir + "_cgo_defun.c")
+		defer fc.Close()
 	}
 	fm := creat(*objDir + "_cgo_main.c")
 
@@ -682,12 +681,11 @@ var isBuiltin = map[string]bool{
 
 func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) {
 	name := n.Mangle
-	if isBuiltin[name] || p.Written[name] {
+	if isBuiltin[name] || !p.Written.Mark(name) {
 		// The builtins are already defined in the C prolog, and we don't
 		// want to duplicate function definitions we've already done.
 		return
 	}
-	p.Written[name] = true
 
 	if *gccgo {
 		p.writeGccgoOutputFunc(fgcc, n)
@@ -1294,7 +1292,7 @@ func (p *Package) writeExportHeader(fgcch io.Writer) {
 
 // gccgoToSymbol converts a name to a mangled symbol for gccgo.
 func gccgoToSymbol(ppath string) string {
-	if gccgoMangler == nil {
+	gccgoManglerOnce.Do(func() {
 		var err error
 		cmd := os.Getenv("GCCGO")
 		if cmd == "" {
@@ -1307,7 +1305,7 @@ func gccgoToSymbol(ppath string) string {
 		if err != nil {
 			fatalf("%v", err)
 		}
-	}
+	})
 	return gccgoMangler(ppath)
 }
 
diff --git a/src/cmd/cgo/util.go b/src/cmd/cgo/util.go
index 00d931b98a0c..bea77b841466 100644
--- a/src/cmd/cgo/util.go
+++ b/src/cmd/cgo/util.go
@@ -11,6 +11,7 @@ import (
 	exec "internal/execabs"
 	"io/ioutil"
 	"os"
+	"sync/atomic"
 )
 
 // run runs the command argv, feeding in stdin on standard input.
@@ -87,16 +88,22 @@ func lineno(pos token.Pos) string {
 func fatalf(msg string, args ...interface{}) {
 	// If we've already printed other errors, they might have
 	// caused the fatal condition. Assume they're enough.
-	if nerrors == 0 {
+	if !nerrors.has() {
 		fmt.Fprintf(os.Stderr, "cgo: "+msg+"\n", args...)
 	}
 	os.Exit(2)
 }
 
-var nerrors int
+type atomicNErrors uint64
+
+var nerrors atomicNErrors
+
+func (n *atomicNErrors) add()        { atomic.AddUint64((*uint64)(n), 1) }
+func (n *atomicNErrors) has() bool   { return n.get() > 0 }
+func (n *atomicNErrors) get() uint64 { return atomic.LoadUint64((*uint64)(n)) }
 
 func error_(pos token.Pos, msg string, args ...interface{}) {
-	nerrors++
+	nerrors.add()
 	if pos.IsValid() {
 		fmt.Fprintf(os.Stderr, "%s: ", fset.Position(pos).String())
 	} else {
diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go
index 0af39a991c02..ac658f4803ef 100644
--- a/src/cmd/go/internal/work/exec.go
+++ b/src/cmd/go/internal/work/exec.go
@@ -2821,7 +2821,12 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo
 		cgoflags = append(cgoflags, "-trimpath", strings.Join(trimpath, ";"))
 	}
 
-	if err := b.run(a, execdir, p.ImportPath, cgoenv, cfg.BuildToolexec, cgoExe, "-objdir", objdir, "-importpath", p.ImportPath, cgoflags, "--", cgoCPPFLAGS, cgoCFLAGS, cgofiles); err != nil {
+	nparallel := runtime.GOMAXPROCS(0)
+	if nparallel > 4 {
+		nparallel = 4
+	}
+
+	if err := b.run(a, execdir, p.ImportPath, cgoenv, cfg.BuildToolexec, cgoExe, "-nparallel", strconv.Itoa(nparallel), "-objdir", objdir, "-importpath", p.ImportPath, cgoflags, "--", cgoCPPFLAGS, cgoCFLAGS, cgofiles); err != nil {
 		return nil, nil, err
 	}
 	outGo = append(outGo, gofiles...)
