884 lines
26 KiB
Diff
884 lines
26 KiB
Diff
From 50e04befeca9ae63296a73c8d5d2870b904971b4 Mon Sep 17 00:00:00 2001
|
|
From: diamondburned <datutbrus@gmail.com>
|
|
Date: Thu, 5 Aug 2021 16:47:49 -0700
|
|
Subject: [PATCH] cmd/cgo: concurrent file generation
|
|
|
|
This commit allows cmd/cgo to generate files in parallel to each other.
|
|
The parallelism is determined by either the thread count (using
|
|
$GOMAXPROCS) or the -nparallel flag, which is set in
|
|
cmd/go/internal/work to be maximum 4.
|
|
|
|
The commit addresses some existing race conditions that had to do with
|
|
mutating the global state by using a global write mutex during the
|
|
translation stage.
|
|
|
|
Other race conditions that had to do with mutating the Package singleton
|
|
was solved by giving each translation worker its own package, and then
|
|
combining them back into a single package.
|
|
|
|
With this and the cmd/go/internal/work changes in place, build time
|
|
(from scratch, with -a) for gotktrix is reduced from 33 minutes to 10
|
|
minutes on an 8-thread Intel Core i5-8250U.
|
|
---
|
|
src/cmd/cgo/gcc.go | 120 +++++++++++-----
|
|
src/cmd/cgo/godefs.go | 6 +-
|
|
src/cmd/cgo/main.go | 238 ++++++++++++++++++++++++-------
|
|
src/cmd/cgo/out.go | 20 ++-
|
|
src/cmd/cgo/util.go | 13 +-
|
|
src/cmd/go/internal/work/exec.go | 7 +-
|
|
6 files changed, 303 insertions(+), 101 deletions(-)
|
|
|
|
diff --git a/src/cmd/cgo/gcc.go b/src/cmd/cgo/gcc.go
|
|
index 997a830994f0..59a8405c0c71 100644
|
|
--- a/src/cmd/cgo/gcc.go
|
|
+++ b/src/cmd/cgo/gcc.go
|
|
@@ -26,6 +26,8 @@ import (
|
|
"os/exec"
|
|
"strconv"
|
|
"strings"
|
|
+ "sync"
|
|
+ "sync/atomic"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
|
|
@@ -485,7 +487,7 @@ func (p *Package) guessKinds(f *File) []*Name {
|
|
}
|
|
needType = append(needType, n)
|
|
}
|
|
- if nerrors > 0 {
|
|
+ if nerrors.has() {
|
|
// Check if compiling the preamble by itself causes any errors,
|
|
// because the messages we've printed out so far aren't helpful
|
|
// to users debugging preamble mistakes. See issue 8442.
|
|
@@ -727,9 +729,12 @@ func (p *Package) prepareNames(f *File) {
|
|
}
|
|
}
|
|
p.mangleName(n)
|
|
+
|
|
+ globalMu.Lock()
|
|
if n.Kind == "type" && typedef[n.Mangle] == nil {
|
|
typedef[n.Mangle] = n.Type
|
|
}
|
|
+ globalMu.Unlock()
|
|
}
|
|
}
|
|
|
|
@@ -1023,7 +1028,7 @@ func (p *Package) hasPointer(f *File, t ast.Expr, top bool) bool {
|
|
}
|
|
// Check whether this is a pointer to a C union (or class)
|
|
// type that contains a pointer.
|
|
- if unionWithPointer[t.X] {
|
|
+ if isUnionWithPointer(t.X) {
|
|
return true
|
|
}
|
|
return p.hasPointer(f, t.X, false)
|
|
@@ -1046,7 +1051,10 @@ func (p *Package) hasPointer(f *File, t ast.Expr, top bool) bool {
|
|
}
|
|
}
|
|
}
|
|
- if def := typedef[t.Name]; def != nil {
|
|
+ globalMu.Lock()
|
|
+ def := typedef[t.Name]
|
|
+ globalMu.Unlock()
|
|
+ if def != nil {
|
|
return p.hasPointer(f, def.Go, top)
|
|
}
|
|
if t.Name == "string" {
|
|
@@ -1582,7 +1590,7 @@ func checkGCCBaseCmd() ([]string, error) {
|
|
}
|
|
|
|
// gccMachine returns the gcc -m flag to use, either "-m32", "-m64" or "-marm".
|
|
-func (p *Package) gccMachine() []string {
|
|
+func gccMachine() []string {
|
|
switch goarch {
|
|
case "amd64":
|
|
if goos == "darwin" {
|
|
@@ -1617,20 +1625,20 @@ func (p *Package) gccMachine() []string {
|
|
return nil
|
|
}
|
|
|
|
-func gccTmp() string {
|
|
- return *objDir + "_cgo_.o"
|
|
+func (p *Package) gccTmp() string {
|
|
+ return fmt.Sprintf("%s_%d_cgo_.o", *objDir, p.workerID)
|
|
}
|
|
|
|
// gccCmd returns the gcc command line to use for compiling
|
|
// the input.
|
|
func (p *Package) gccCmd() []string {
|
|
c := append(gccBaseCmd,
|
|
- "-w", // no warnings
|
|
- "-Wno-error", // warnings are not errors
|
|
- "-o"+gccTmp(), // write object to tmp
|
|
- "-gdwarf-2", // generate DWARF v2 debugging symbols
|
|
- "-c", // do not link
|
|
- "-xc", // input language is C
|
|
+ "-w", // no warnings
|
|
+ "-Wno-error", // warnings are not errors
|
|
+ "-o"+p.gccTmp(), // write object to tmp
|
|
+ "-gdwarf-2", // generate DWARF v2 debugging symbols
|
|
+ "-c", // do not link
|
|
+ "-xc", // input language is C
|
|
)
|
|
if p.GccIsClang {
|
|
c = append(c,
|
|
@@ -1653,7 +1661,7 @@ func (p *Package) gccCmd() []string {
|
|
}
|
|
|
|
c = append(c, p.GccOptions...)
|
|
- c = append(c, p.gccMachine()...)
|
|
+ c = append(c, gccMachine()...)
|
|
if goos == "aix" {
|
|
c = append(c, "-maix64")
|
|
c = append(c, "-mcmodel=large")
|
|
@@ -1717,11 +1725,11 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
|
|
}
|
|
}
|
|
|
|
- if f, err := macho.Open(gccTmp()); err == nil {
|
|
+ if f, err := macho.Open(p.gccTmp()); err == nil {
|
|
defer f.Close()
|
|
d, err := f.DWARF()
|
|
if err != nil {
|
|
- fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
|
|
+ fatalf("cannot load DWARF output from %s: %v", p.gccTmp(), err)
|
|
}
|
|
bo := f.ByteOrder
|
|
if f.Symtab != nil {
|
|
@@ -1795,11 +1803,11 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
|
|
return d, ints, floats, strs
|
|
}
|
|
|
|
- if f, err := elf.Open(gccTmp()); err == nil {
|
|
+ if f, err := elf.Open(p.gccTmp()); err == nil {
|
|
defer f.Close()
|
|
d, err := f.DWARF()
|
|
if err != nil {
|
|
- fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
|
|
+ fatalf("cannot load DWARF output from %s: %v", p.gccTmp(), err)
|
|
}
|
|
bo := f.ByteOrder
|
|
symtab, err := f.Symbols()
|
|
@@ -1874,11 +1882,11 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
|
|
return d, ints, floats, strs
|
|
}
|
|
|
|
- if f, err := pe.Open(gccTmp()); err == nil {
|
|
+ if f, err := pe.Open(p.gccTmp()); err == nil {
|
|
defer f.Close()
|
|
d, err := f.DWARF()
|
|
if err != nil {
|
|
- fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
|
|
+ fatalf("cannot load DWARF output from %s: %v", p.gccTmp(), err)
|
|
}
|
|
bo := binary.LittleEndian
|
|
for _, s := range f.Symbols {
|
|
@@ -1946,11 +1954,11 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
|
|
return d, ints, floats, strs
|
|
}
|
|
|
|
- if f, err := xcoff.Open(gccTmp()); err == nil {
|
|
+ if f, err := xcoff.Open(p.gccTmp()); err == nil {
|
|
defer f.Close()
|
|
d, err := f.DWARF()
|
|
if err != nil {
|
|
- fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
|
|
+ fatalf("cannot load DWARF output from %s: %v", p.gccTmp(), err)
|
|
}
|
|
bo := binary.BigEndian
|
|
for _, s := range f.Symbols {
|
|
@@ -2016,7 +2024,7 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
|
|
buildStrings()
|
|
return d, ints, floats, strs
|
|
}
|
|
- fatalf("cannot parse gcc output %s as ELF, Mach-O, PE, XCOFF object", gccTmp())
|
|
+ fatalf("cannot parse gcc output %s as ELF, Mach-O, PE, XCOFF object", p.gccTmp())
|
|
panic("not reached")
|
|
}
|
|
|
|
@@ -2026,7 +2034,7 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
|
|
// and its included files.
|
|
func (p *Package) gccDefines(stdin []byte) string {
|
|
base := append(gccBaseCmd, "-E", "-dM", "-xc")
|
|
- base = append(base, p.gccMachine()...)
|
|
+ base = append(base, gccMachine()...)
|
|
stdout, _ := runGcc(stdin, append(append(base, p.GccOptions...), "-"))
|
|
return stdout
|
|
}
|
|
@@ -2126,7 +2134,13 @@ type typeConv struct {
|
|
intSize int64
|
|
}
|
|
|
|
-var tagGen int
|
|
+var tagGenCounter int64
|
|
+
|
|
+func genTag() int {
|
|
+ return int(atomic.AddInt64(&tagGenCounter, 1) - 1)
|
|
+}
|
|
+
|
|
+var globalMu sync.Mutex
|
|
var typedef = make(map[string]*Type)
|
|
var goIdent = make(map[string]*ast.Ident)
|
|
|
|
@@ -2134,6 +2148,29 @@ var goIdent = make(map[string]*ast.Ident)
|
|
// that may contain a pointer. This is used for cgo pointer checking.
|
|
var unionWithPointer = make(map[ast.Expr]bool)
|
|
|
|
+func unionWithPointerTrueIfKey(setKey, ifKey ast.Expr) {
|
|
+ globalMu.Lock()
|
|
+ defer globalMu.Unlock()
|
|
+
|
|
+ if unionWithPointer[ifKey] {
|
|
+ unionWithPointer[setKey] = true
|
|
+ }
|
|
+}
|
|
+
|
|
+func isUnionWithPointer(key ast.Expr) bool {
|
|
+ globalMu.Lock()
|
|
+ defer globalMu.Unlock()
|
|
+
|
|
+ return unionWithPointer[key]
|
|
+}
|
|
+
|
|
+func unionWithPointerTrue(key ast.Expr) {
|
|
+ globalMu.Lock()
|
|
+ defer globalMu.Unlock()
|
|
+
|
|
+ unionWithPointer[key] = true
|
|
+}
|
|
+
|
|
// anonymousStructTag provides a consistent tag for an anonymous struct.
|
|
// The same dwarf.StructType pointer will always get the same tag.
|
|
var anonymousStructTag = make(map[*dwarf.StructType]string)
|
|
@@ -2472,9 +2509,7 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
|
|
t.Size = t1.Size
|
|
t.Align = t1.Align
|
|
t.Go = t1.Go
|
|
- if unionWithPointer[t1.Go] {
|
|
- unionWithPointer[t.Go] = true
|
|
- }
|
|
+ unionWithPointerTrueIfKey(t.Go, t1.Go)
|
|
t.EnumValues = nil
|
|
t.Typedef = ""
|
|
t.C.Set("%s "+dt.Qual, t1.C)
|
|
@@ -2488,18 +2523,21 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
|
|
break
|
|
}
|
|
if tag == "" {
|
|
+ globalMu.Lock()
|
|
tag = anonymousStructTag[dt]
|
|
if tag == "" {
|
|
- tag = "__" + strconv.Itoa(tagGen)
|
|
- tagGen++
|
|
+ tag = "__" + strconv.Itoa(genTag())
|
|
anonymousStructTag[dt] = tag
|
|
}
|
|
+ globalMu.Unlock()
|
|
} else if t.C.Empty() {
|
|
t.C.Set(dt.Kind + " " + tag)
|
|
}
|
|
name := c.Ident("_Ctype_" + dt.Kind + "_" + tag)
|
|
t.Go = name // publish before recursive calls
|
|
+ globalMu.Lock()
|
|
goIdent[name.Name] = name
|
|
+ globalMu.Unlock()
|
|
if dt.ByteSize < 0 {
|
|
// Size calculation in c.Struct/c.Opaque will die with size=-1 (unknown),
|
|
// so execute the basic things that the struct case would do
|
|
@@ -2519,20 +2557,24 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
|
|
// on the Go heap, right? It currently doesn't work for unions because
|
|
// they are defined as a type alias for struct{}, not a defined type.
|
|
}
|
|
+ globalMu.Lock()
|
|
typedef[name.Name] = &tt
|
|
+ globalMu.Unlock()
|
|
break
|
|
}
|
|
switch dt.Kind {
|
|
case "class", "union":
|
|
t.Go = c.Opaque(t.Size)
|
|
if c.dwarfHasPointer(dt, pos) {
|
|
- unionWithPointer[t.Go] = true
|
|
+ unionWithPointerTrue(t.Go)
|
|
}
|
|
if t.C.Empty() {
|
|
t.C.Set("__typeof__(unsigned char[%d])", t.Size)
|
|
}
|
|
t.Align = 1 // TODO: should probably base this on field alignment.
|
|
+ globalMu.Lock()
|
|
typedef[name.Name] = t
|
|
+ globalMu.Unlock()
|
|
case "struct":
|
|
g, csyntax, align := c.Struct(dt, pos)
|
|
if t.C.Empty() {
|
|
@@ -2545,7 +2587,9 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
|
|
}
|
|
tt.Go = g
|
|
tt.NotInHeap = c.notInHeapStructs[tag]
|
|
+ globalMu.Lock()
|
|
typedef[name.Name] = &tt
|
|
+ globalMu.Unlock()
|
|
}
|
|
|
|
case *dwarf.TypedefType:
|
|
@@ -2568,7 +2612,9 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
|
|
break
|
|
}
|
|
name := c.Ident("_Ctype_" + dt.Name)
|
|
+ globalMu.Lock()
|
|
goIdent[name.Name] = name
|
|
+ globalMu.Unlock()
|
|
akey := ""
|
|
if c.anonymousStructTypedef(dt) {
|
|
// only load type recursively for typedefs of anonymous
|
|
@@ -2583,10 +2629,12 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
|
|
s.BadPointer = true
|
|
sub = &s
|
|
// Make sure we update any previously computed type.
|
|
+ globalMu.Lock()
|
|
if oldType := typedef[name.Name]; oldType != nil {
|
|
oldType.Go = sub.Go
|
|
oldType.BadPointer = true
|
|
}
|
|
+ globalMu.Unlock()
|
|
}
|
|
if c.badVoidPointerTypedef(dt) {
|
|
// Treat this typedef as a pointer to a NotInHeap void.
|
|
@@ -2615,11 +2663,10 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
|
|
t.Go = name
|
|
t.BadPointer = sub.BadPointer
|
|
t.NotInHeap = sub.NotInHeap
|
|
- if unionWithPointer[sub.Go] {
|
|
- unionWithPointer[t.Go] = true
|
|
- }
|
|
+ unionWithPointerTrueIfKey(t.Go, sub.Go)
|
|
t.Size = sub.Size
|
|
t.Align = sub.Align
|
|
+ globalMu.Lock()
|
|
oldType := typedef[name.Name]
|
|
if oldType == nil {
|
|
tt := *t
|
|
@@ -2628,6 +2675,7 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
|
|
tt.NotInHeap = sub.NotInHeap
|
|
typedef[name.Name] = &tt
|
|
}
|
|
+ globalMu.Unlock()
|
|
|
|
// If sub.Go.Name is "_Ctype_struct_foo" or "_Ctype_union_foo" or "_Ctype_class_foo",
|
|
// use that as the Go form for this typedef too, so that the typedef will be interchangeable
|
|
@@ -2638,7 +2686,9 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
|
|
|
|
if isStructUnionClass(sub.Go) {
|
|
// Use the typedef name for C code.
|
|
+ globalMu.Lock()
|
|
typedef[sub.Go.(*ast.Ident).Name].C = t.C
|
|
+ globalMu.Unlock()
|
|
}
|
|
|
|
// If we've seen this typedef before, and it
|
|
@@ -2698,8 +2748,12 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
|
|
}
|
|
s = strings.Replace(s, " ", "", -1)
|
|
name := c.Ident("_Ctype_" + s)
|
|
+
|
|
tt := *t
|
|
+ globalMu.Lock()
|
|
typedef[name.Name] = &tt
|
|
+ globalMu.Unlock()
|
|
+
|
|
if !*godefs {
|
|
t.Go = name
|
|
}
|
|
diff --git a/src/cmd/cgo/godefs.go b/src/cmd/cgo/godefs.go
|
|
index c0d59aee01d7..740aa3bc30b2 100644
|
|
--- a/src/cmd/cgo/godefs.go
|
|
+++ b/src/cmd/cgo/godefs.go
|
|
@@ -115,11 +115,11 @@ func (p *Package) godefs(f *File) string {
|
|
return buf.String()
|
|
}
|
|
|
|
-var gofmtBuf bytes.Buffer
|
|
-
|
|
// gofmt returns the gofmt-formatted string for an AST node.
|
|
func gofmt(n interface{}) string {
|
|
- gofmtBuf.Reset()
|
|
+ var gofmtBuf strings.Builder
|
|
+ gofmtBuf.Grow(512)
|
|
+
|
|
err := printer.Fprint(&gofmtBuf, fset, n)
|
|
if err != nil {
|
|
return "<" + err.Error() + ">"
|
|
diff --git a/src/cmd/cgo/main.go b/src/cmd/cgo/main.go
|
|
index 14642b7576b0..36e76908c602 100644
|
|
--- a/src/cmd/cgo/main.go
|
|
+++ b/src/cmd/cgo/main.go
|
|
@@ -26,6 +26,7 @@ import (
|
|
"runtime"
|
|
"sort"
|
|
"strings"
|
|
+ "sync"
|
|
|
|
"cmd/internal/edit"
|
|
"cmd/internal/objabi"
|
|
@@ -33,6 +34,8 @@ import (
|
|
|
|
// A Package collects information about the package we're going to write.
|
|
type Package struct {
|
|
+ workerID int
|
|
+
|
|
PackageName string // name of package
|
|
PackagePath string
|
|
PtrSize int64
|
|
@@ -40,17 +43,36 @@ type Package struct {
|
|
GccOptions []string
|
|
GccIsClang bool
|
|
CgoFlags map[string][]string // #cgo flags (CFLAGS, LDFLAGS)
|
|
- Written map[string]bool
|
|
- Name map[string]*Name // accumulated Name from Files
|
|
+ Written *WrittenFiles
|
|
+ Name map[string]*Name // TODO: parallelize. accumulated Name from Files
|
|
ExpFunc []*ExpFunc // accumulated ExpFunc from Files
|
|
- Decl []ast.Decl
|
|
- GoFiles []string // list of Go files
|
|
- GccFiles []string // list of gcc output files
|
|
- Preamble string // collected preamble for _cgo_export.h
|
|
+ Decl []ast.Decl `json:"-"`
|
|
+ GoFiles []string // list of Go files
|
|
+ GccFiles []string // list of gcc output files
|
|
+ Preamble string // collected preamble for _cgo_export.h
|
|
+
|
|
typedefs map[string]bool // type names that appear in the types of the objects we're interested in
|
|
typedefList []typedefInfo
|
|
}
|
|
|
|
+type WrittenFiles struct {
|
|
+ mut sync.Mutex
|
|
+ written map[string]struct{}
|
|
+}
|
|
+
|
|
+func (w *WrittenFiles) Mark(file string) (shouldWrite bool) {
|
|
+ w.mut.Lock()
|
|
+ defer w.mut.Unlock()
|
|
+
|
|
+ _, already := w.written[file]
|
|
+ if already {
|
|
+ return false
|
|
+ }
|
|
+
|
|
+ w.written[file] = struct{}{}
|
|
+ return true
|
|
+}
|
|
+
|
|
// A typedefInfo is an element on Package.typedefList: a typedef name
|
|
// and the position where it was required.
|
|
type typedefInfo struct {
|
|
@@ -87,7 +109,7 @@ func nameKeys(m map[string]*Name) []string {
|
|
|
|
// A Call refers to a call of a C.xxx function in the AST.
|
|
type Call struct {
|
|
- Call *ast.CallExpr
|
|
+ Call *ast.CallExpr `json:"-"`
|
|
Deferred bool
|
|
Done bool
|
|
}
|
|
@@ -95,7 +117,7 @@ type Call struct {
|
|
// A Ref refers to an expression of the form C.xxx in the AST.
|
|
type Ref struct {
|
|
Name *Name
|
|
- Expr *ast.Expr
|
|
+ Expr *ast.Expr `json:"-"`
|
|
Context astContext
|
|
Done bool
|
|
}
|
|
@@ -108,13 +130,13 @@ var nameKinds = []string{"iconst", "fconst", "sconst", "type", "var", "fpvar", "
|
|
|
|
// A Name collects information about C.xxx.
|
|
type Name struct {
|
|
- Go string // name used in Go referring to package C
|
|
- Mangle string // name used in generated Go
|
|
- C string // name used in C
|
|
- Define string // #define expansion
|
|
- Kind string // one of the nameKinds
|
|
- Type *Type // the type of xxx
|
|
- FuncType *FuncType
|
|
+ Go string // name used in Go referring to package C
|
|
+ Mangle string // name used in generated Go
|
|
+ C string // name used in C
|
|
+ Define string // #define expansion
|
|
+ Kind string // one of the nameKinds
|
|
+ Type *Type `json:"-"` // the type of xxx
|
|
+ FuncType *FuncType `json:"-"`
|
|
AddError bool
|
|
Const string // constant definition
|
|
}
|
|
@@ -133,23 +155,23 @@ func (n *Name) IsConst() bool {
|
|
// Such functions are identified in the Go input file
|
|
// by doc comments containing the line //export ExpName
|
|
type ExpFunc struct {
|
|
- Func *ast.FuncDecl
|
|
- ExpName string // name to use from C
|
|
+ Func *ast.FuncDecl `json:"-"`
|
|
+ ExpName string // name to use from C
|
|
Doc string
|
|
}
|
|
|
|
// A TypeRepr contains the string representation of a type.
|
|
type TypeRepr struct {
|
|
Repr string
|
|
- FormatArgs []interface{}
|
|
+ FormatArgs []interface{} `json:"-"`
|
|
}
|
|
|
|
// A Type collects information about a type in both the C and Go worlds.
|
|
type Type struct {
|
|
Size int64
|
|
Align int64
|
|
- C *TypeRepr
|
|
- Go ast.Expr
|
|
+ C *TypeRepr `json:"-"`
|
|
+ Go ast.Expr `json:"-"`
|
|
EnumValues map[string]int64
|
|
Typedef string
|
|
BadPointer bool // this pointer type should be represented as a uintptr (deprecated)
|
|
@@ -223,6 +245,8 @@ var cPrefix string
|
|
|
|
var fset = token.NewFileSet()
|
|
|
|
+var nparallel = flag.Int("nparallel", runtime.GOMAXPROCS(0), "maximum parallel jobs")
|
|
+
|
|
var dynobj = flag.String("dynimport", "", "if non-empty, print dynamic import data for that file")
|
|
var dynout = flag.String("dynout", "", "write -dynimport output to this file")
|
|
var dynpackage = flag.String("dynpackage", "main", "set Go package for -dynimport output")
|
|
@@ -241,11 +265,13 @@ var exportHeader = flag.String("exportheader", "", "where to write export header
|
|
var gccgo = flag.Bool("gccgo", false, "generate files for use with gccgo")
|
|
var gccgoprefix = flag.String("gccgoprefix", "", "-fgo-prefix option used with gccgo")
|
|
var gccgopkgpath = flag.String("gccgopkgpath", "", "-fgo-pkgpath option used with gccgo")
|
|
-var gccgoMangler func(string) string
|
|
var importRuntimeCgo = flag.Bool("import_runtime_cgo", true, "import runtime/cgo in generated code")
|
|
var importSyscall = flag.Bool("import_syscall", true, "import syscall in generated code")
|
|
var trimpath = flag.String("trimpath", "", "applies supplied rewrites or trims prefixes to recorded source file paths")
|
|
|
|
+var gccgoMangler func(string) string
|
|
+var gccgoManglerOnce sync.Once
|
|
+
|
|
var goarch, goos, gomips, gomips64 string
|
|
var gccBaseCmd []string
|
|
|
|
@@ -302,7 +328,7 @@ func main() {
|
|
}
|
|
}
|
|
|
|
- p := newPackage(args[:i])
|
|
+ p := newPackage(args[:i], 0)
|
|
|
|
// We need a C compiler to be available. Check this.
|
|
var err error
|
|
@@ -370,43 +396,152 @@ func main() {
|
|
}
|
|
*objDir += string(filepath.Separator)
|
|
|
|
- for i, input := range goFiles {
|
|
- f := fs[i]
|
|
- p.Translate(f)
|
|
- for _, cref := range f.Ref {
|
|
- switch cref.Context {
|
|
- case ctxCall, ctxCall2:
|
|
- if cref.Name.Kind != "type" {
|
|
- break
|
|
- }
|
|
- old := *cref.Expr
|
|
- *cref.Expr = cref.Name.Type.Go
|
|
- f.Edit.Replace(f.offset(old.Pos()), f.offset(old.End()), gofmt(cref.Name.Type.Go))
|
|
- }
|
|
- }
|
|
- if nerrors > 0 {
|
|
- os.Exit(2)
|
|
- }
|
|
- p.PackagePath = f.Package
|
|
- p.Record(f)
|
|
- if *godefs {
|
|
- os.Stdout.WriteString(p.godefs(f))
|
|
- } else {
|
|
- p.writeOutput(f, input)
|
|
+ pkgs := doFiles(p, goFiles, fs)
|
|
+
|
|
+ if !*godefs {
|
|
+ pkg := coalescePackages(pkgs)
|
|
+ pkg.writeDefs()
|
|
+ }
|
|
+
|
|
+ if nerrors.get() > 0 {
|
|
+ os.Exit(2)
|
|
+ }
|
|
+}
|
|
+
|
|
+type doFileJob struct {
|
|
+ input string
|
|
+ file *File
|
|
+}
|
|
+
|
|
+func doFiles(p *Package, inputs []string, files []*File) []*Package {
|
|
+ if len(inputs) != len(files) {
|
|
+ fatalf("unexpected len(inputs) %d != len(files) %d", len(inputs), len(files))
|
|
+ }
|
|
+
|
|
+ jobCh := make(chan doFileJob)
|
|
+
|
|
+ var wg sync.WaitGroup
|
|
+ wg.Add(*nparallel)
|
|
+
|
|
+ pkgs := make([]*Package, *nparallel)
|
|
+ pkgs[0] = p
|
|
+
|
|
+ for n := 1; n < *nparallel; n++ {
|
|
+ pkgs[n] = cloneNewPackage(p, n)
|
|
+ }
|
|
+
|
|
+ for n := 0; n < *nparallel; n++ {
|
|
+ p := pkgs[n]
|
|
+ go func() {
|
|
+ doFileWorker(p, jobCh)
|
|
+ wg.Done()
|
|
+ }()
|
|
+ }
|
|
+
|
|
+ for i := range inputs {
|
|
+ jobCh <- doFileJob{
|
|
+ input: inputs[i],
|
|
+ file: files[i],
|
|
}
|
|
}
|
|
|
|
- if !*godefs {
|
|
- p.writeDefs()
|
|
+ close(jobCh)
|
|
+ wg.Wait()
|
|
+
|
|
+ return pkgs
|
|
+}
|
|
+
|
|
+func doFileWorker(p *Package, jobCh <-chan doFileJob) *Package {
|
|
+ for job := range jobCh {
|
|
+ doFile(p, job.input, job.file)
|
|
}
|
|
- if nerrors > 0 {
|
|
+
|
|
+ return p
|
|
+}
|
|
+
|
|
+func doFile(p *Package, input string, f *File) {
|
|
+ p.Translate(f)
|
|
+ for _, cref := range f.Ref {
|
|
+ switch cref.Context {
|
|
+ case ctxCall, ctxCall2:
|
|
+ if cref.Name.Kind != "type" {
|
|
+ break
|
|
+ }
|
|
+ old := *cref.Expr
|
|
+ *cref.Expr = cref.Name.Type.Go
|
|
+ f.Edit.Replace(f.offset(old.Pos()), f.offset(old.End()), gofmt(cref.Name.Type.Go))
|
|
+ }
|
|
+ }
|
|
+ if nerrors.get() > 0 {
|
|
os.Exit(2)
|
|
}
|
|
+ p.PackagePath = f.Package
|
|
+ p.Record(f)
|
|
+ if *godefs {
|
|
+ os.Stdout.WriteString(p.godefs(f))
|
|
+ } else {
|
|
+ p.writeOutput(f, input)
|
|
+ }
|
|
+}
|
|
+
|
|
+func cloneNewPackage(p *Package, workerID int) *Package {
|
|
+ return &Package{
|
|
+ workerID: workerID,
|
|
+ PackageName: p.PackageName,
|
|
+ PackagePath: p.PackagePath,
|
|
+ PtrSize: p.PtrSize,
|
|
+ IntSize: p.IntSize,
|
|
+ GccOptions: p.GccOptions, // init in main only
|
|
+ GccIsClang: p.GccIsClang,
|
|
+ CgoFlags: p.CgoFlags, // init in main only
|
|
+ Written: p.Written,
|
|
+ }
|
|
+}
|
|
+
|
|
+func coalescePackages(pkgs []*Package) *Package {
|
|
+ if len(pkgs) == 0 {
|
|
+ return nil
|
|
+ }
|
|
+
|
|
+ nonEmpty := pkgs[:0]
|
|
+ for _, p := range pkgs {
|
|
+ if p.PackageName != "" {
|
|
+ nonEmpty = append(nonEmpty, p)
|
|
+ }
|
|
+ }
|
|
+
|
|
+ pkgs = nonEmpty
|
|
+ pkg := pkgs[0]
|
|
+
|
|
+ for _, p := range pkgs[1:] {
|
|
+ pkg.PackageName = p.PackageName
|
|
+ pkg.PackagePath = p.PackagePath
|
|
+ pkg.ExpFunc = append(pkg.ExpFunc, p.ExpFunc...)
|
|
+ pkg.Decl = append(pkg.Decl, p.Decl...)
|
|
+ pkg.GoFiles = append(pkg.GoFiles, p.GoFiles...)
|
|
+ pkg.GccFiles = append(pkg.GccFiles, p.GccFiles...)
|
|
+ pkg.Preamble += p.Preamble
|
|
+
|
|
+ if pkg.Name == nil && p.Name != nil {
|
|
+ pkg.Name = p.Name
|
|
+ continue
|
|
+ }
|
|
+
|
|
+ for k, v := range p.Name {
|
|
+ if o, ok := pkg.Name[k]; ok && !reflect.DeepEqual(v, o) {
|
|
+ fatalf("duplicate pkg.Name %s, %v == %v", k, v, o)
|
|
+ }
|
|
+
|
|
+ pkg.Name[k] = v
|
|
+ }
|
|
+ }
|
|
+
|
|
+ return pkg
|
|
}
|
|
|
|
// newPackage returns a new Package that will invoke
|
|
// gcc with the additional arguments specified in args.
|
|
-func newPackage(args []string) *Package {
|
|
+func newPackage(args []string, workerID int) *Package {
|
|
goarch = runtime.GOARCH
|
|
if s := os.Getenv("GOARCH"); s != "" {
|
|
goarch = s
|
|
@@ -432,10 +567,13 @@ func newPackage(args []string) *Package {
|
|
os.Setenv("LC_ALL", "C")
|
|
|
|
p := &Package{
|
|
+ workerID: workerID,
|
|
PtrSize: ptrSize,
|
|
IntSize: intSize,
|
|
CgoFlags: make(map[string][]string),
|
|
- Written: make(map[string]bool),
|
|
+ Written: &WrittenFiles{
|
|
+ written: make(map[string]struct{}),
|
|
+ },
|
|
}
|
|
p.addToFlag("CFLAGS", args)
|
|
return p
|
|
diff --git a/src/cmd/cgo/out.go b/src/cmd/cgo/out.go
|
|
index 4968f7059d9b..b934d12d9423 100644
|
|
--- a/src/cmd/cgo/out.go
|
|
+++ b/src/cmd/cgo/out.go
|
|
@@ -32,14 +32,13 @@ var (
|
|
|
|
// writeDefs creates output files to be compiled by gc and gcc.
|
|
func (p *Package) writeDefs() {
|
|
- var fgo2, fc io.Writer
|
|
- f := creat(*objDir + "_cgo_gotypes.go")
|
|
- defer f.Close()
|
|
- fgo2 = f
|
|
+ fgo2 := creat(*objDir + "_cgo_gotypes.go")
|
|
+ defer fgo2.Close()
|
|
+
|
|
+ var fc *os.File
|
|
if *gccgo {
|
|
- f := creat(*objDir + "_cgo_defun.c")
|
|
- defer f.Close()
|
|
- fc = f
|
|
+ fc = creat(*objDir + "_cgo_defun.c")
|
|
+ defer fc.Close()
|
|
}
|
|
fm := creat(*objDir + "_cgo_main.c")
|
|
|
|
@@ -682,12 +681,11 @@ var isBuiltin = map[string]bool{
|
|
|
|
func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) {
|
|
name := n.Mangle
|
|
- if isBuiltin[name] || p.Written[name] {
|
|
+ if isBuiltin[name] || !p.Written.Mark(name) {
|
|
// The builtins are already defined in the C prolog, and we don't
|
|
// want to duplicate function definitions we've already done.
|
|
return
|
|
}
|
|
- p.Written[name] = true
|
|
|
|
if *gccgo {
|
|
p.writeGccgoOutputFunc(fgcc, n)
|
|
@@ -1294,7 +1292,7 @@ func (p *Package) writeExportHeader(fgcch io.Writer) {
|
|
|
|
// gccgoToSymbol converts a name to a mangled symbol for gccgo.
|
|
func gccgoToSymbol(ppath string) string {
|
|
- if gccgoMangler == nil {
|
|
+ gccgoManglerOnce.Do(func() {
|
|
var err error
|
|
cmd := os.Getenv("GCCGO")
|
|
if cmd == "" {
|
|
@@ -1307,7 +1305,7 @@ func gccgoToSymbol(ppath string) string {
|
|
if err != nil {
|
|
fatalf("%v", err)
|
|
}
|
|
- }
|
|
+ })
|
|
return gccgoMangler(ppath)
|
|
}
|
|
|
|
diff --git a/src/cmd/cgo/util.go b/src/cmd/cgo/util.go
|
|
index 00d931b98a0c..bea77b841466 100644
|
|
--- a/src/cmd/cgo/util.go
|
|
+++ b/src/cmd/cgo/util.go
|
|
@@ -11,6 +11,7 @@ import (
|
|
exec "internal/execabs"
|
|
"io/ioutil"
|
|
"os"
|
|
+ "sync/atomic"
|
|
)
|
|
|
|
// run runs the command argv, feeding in stdin on standard input.
|
|
@@ -87,16 +88,22 @@ func lineno(pos token.Pos) string {
|
|
func fatalf(msg string, args ...interface{}) {
|
|
// If we've already printed other errors, they might have
|
|
// caused the fatal condition. Assume they're enough.
|
|
- if nerrors == 0 {
|
|
+ if !nerrors.has() {
|
|
fmt.Fprintf(os.Stderr, "cgo: "+msg+"\n", args...)
|
|
}
|
|
os.Exit(2)
|
|
}
|
|
|
|
-var nerrors int
|
|
+type atomicNErrors uint64
|
|
+
|
|
+var nerrors atomicNErrors
|
|
+
|
|
+func (n *atomicNErrors) add() { atomic.AddUint64((*uint64)(n), 1) }
|
|
+func (n *atomicNErrors) has() bool { return n.get() > 0 }
|
|
+func (n *atomicNErrors) get() uint64 { return atomic.LoadUint64((*uint64)(n)) }
|
|
|
|
func error_(pos token.Pos, msg string, args ...interface{}) {
|
|
- nerrors++
|
|
+ nerrors.add()
|
|
if pos.IsValid() {
|
|
fmt.Fprintf(os.Stderr, "%s: ", fset.Position(pos).String())
|
|
} else {
|
|
diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go
|
|
index 0af39a991c02..ac658f4803ef 100644
|
|
--- a/src/cmd/go/internal/work/exec.go
|
|
+++ b/src/cmd/go/internal/work/exec.go
|
|
@@ -2821,7 +2821,12 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo
|
|
cgoflags = append(cgoflags, "-trimpath", strings.Join(trimpath, ";"))
|
|
}
|
|
|
|
- if err := b.run(a, execdir, p.ImportPath, cgoenv, cfg.BuildToolexec, cgoExe, "-objdir", objdir, "-importpath", p.ImportPath, cgoflags, "--", cgoCPPFLAGS, cgoCFLAGS, cgofiles); err != nil {
|
|
+ nparallel := runtime.GOMAXPROCS(0)
|
|
+ if nparallel > 4 {
|
|
+ nparallel = 4
|
|
+ }
|
|
+
|
|
+ if err := b.run(a, execdir, p.ImportPath, cgoenv, cfg.BuildToolexec, cgoExe, "-nparallel", strconv.Itoa(nparallel), "-objdir", objdir, "-importpath", p.ImportPath, cgoflags, "--", cgoCPPFLAGS, cgoCFLAGS, cgofiles); err != nil {
|
|
return nil, nil, err
|
|
}
|
|
outGo = append(outGo, gofiles...)
|