wanderer/math-optim
1
1
Fork 0
Code Issues 1 Pull Requests Activity
04ad04188a
math-optim/patches/diamondburned/go/50e04befeca9ae63296a73c8d5d2870b904971b4.patch

884 lines
26 KiB
Diff
Raw Normal View History

initial commit
2022-06-14 16:46:11 +02:00
From 50e04befeca9ae63296a73c8d5d2870b904971b4 Mon Sep 17 00:00:00 2001
From: diamondburned <datutbrus@gmail.com>
Date: Thu, 5 Aug 2021 16:47:49 -0700
Subject: [PATCH] cmd/cgo: concurrent file generation
This commit allows cmd/cgo to generate files in parallel to each other.
The parallelism is determined by either the thread count (using
$GOMAXPROCS) or the -nparallel flag, which is set in
cmd/go/internal/work to be maximum 4.
The commit addresses some existing race conditions that had to do with
mutating the global state by using a global write mutex during the
translation stage.
Other race conditions that had to do with mutating the Package singleton
was solved by giving each translation worker its own package, and then
combining them back into a single package.
With this and the cmd/go/internal/work changes in place, build time
(from scratch, with -a) for gotktrix is reduced from 33 minutes to 10
minutes on an 8-thread Intel Core i5-8250U.
---
src/cmd/cgo/gcc.go | 120 +++++++++++-----
src/cmd/cgo/godefs.go | 6 +-
src/cmd/cgo/main.go | 238 ++++++++++++++++++++++++-------
src/cmd/cgo/out.go | 20 ++-
src/cmd/cgo/util.go | 13 +-
src/cmd/go/internal/work/exec.go | 7 +-
6 files changed, 303 insertions(+), 101 deletions(-)
diff --git a/src/cmd/cgo/gcc.go b/src/cmd/cgo/gcc.go
index 997a830994f0..59a8405c0c71 100644
--- a/src/cmd/cgo/gcc.go
+++ b/src/cmd/cgo/gcc.go
@@ -26,6 +26,8 @@ import (
"os/exec"
"strconv"
"strings"
+ "sync"
+ "sync/atomic"
"unicode"
"unicode/utf8"
@@ -485,7 +487,7 @@ func (p *Package) guessKinds(f *File) []*Name {
}
needType = append(needType, n)
}
- if nerrors > 0 {
+ if nerrors.has() {
// Check if compiling the preamble by itself causes any errors,
// because the messages we've printed out so far aren't helpful
// to users debugging preamble mistakes. See issue 8442.
@@ -727,9 +729,12 @@ func (p *Package) prepareNames(f *File) {
}
}
p.mangleName(n)
+
+ globalMu.Lock()
if n.Kind == "type" && typedef[n.Mangle] == nil {
typedef[n.Mangle] = n.Type
}
+ globalMu.Unlock()
}
}
@@ -1023,7 +1028,7 @@ func (p *Package) hasPointer(f *File, t ast.Expr, top bool) bool {
}
// Check whether this is a pointer to a C union (or class)
// type that contains a pointer.
- if unionWithPointer[t.X] {
+ if isUnionWithPointer(t.X) {
return true
}
return p.hasPointer(f, t.X, false)
@@ -1046,7 +1051,10 @@ func (p *Package) hasPointer(f *File, t ast.Expr, top bool) bool {
}
}
}
- if def := typedef[t.Name]; def != nil {
+ globalMu.Lock()
+ def := typedef[t.Name]
+ globalMu.Unlock()
+ if def != nil {
return p.hasPointer(f, def.Go, top)
}
if t.Name == "string" {
@@ -1582,7 +1590,7 @@ func checkGCCBaseCmd() ([]string, error) {
}
// gccMachine returns the gcc -m flag to use, either "-m32", "-m64" or "-marm".
-func (p *Package) gccMachine() []string {
+func gccMachine() []string {
switch goarch {
case "amd64":
if goos == "darwin" {
@@ -1617,20 +1625,20 @@ func (p *Package) gccMachine() []string {
return nil
}
-func gccTmp() string {
- return *objDir + "_cgo_.o"
+func (p *Package) gccTmp() string {
+ return fmt.Sprintf("%s_%d_cgo_.o", *objDir, p.workerID)
}
// gccCmd returns the gcc command line to use for compiling
// the input.
func (p *Package) gccCmd() []string {
c := append(gccBaseCmd,
- "-w", // no warnings
- "-Wno-error", // warnings are not errors
- "-o"+gccTmp(), // write object to tmp
- "-gdwarf-2", // generate DWARF v2 debugging symbols
- "-c", // do not link
- "-xc", // input language is C
+ "-w", // no warnings
+ "-Wno-error", // warnings are not errors
+ "-o"+p.gccTmp(), // write object to tmp
+ "-gdwarf-2", // generate DWARF v2 debugging symbols
+ "-c", // do not link
+ "-xc", // input language is C
)
if p.GccIsClang {
c = append(c,
@@ -1653,7 +1661,7 @@ func (p *Package) gccCmd() []string {
}
c = append(c, p.GccOptions...)
- c = append(c, p.gccMachine()...)
+ c = append(c, gccMachine()...)
if goos == "aix" {
c = append(c, "-maix64")
c = append(c, "-mcmodel=large")
@@ -1717,11 +1725,11 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
}
}
- if f, err := macho.Open(gccTmp()); err == nil {
+ if f, err := macho.Open(p.gccTmp()); err == nil {
defer f.Close()
d, err := f.DWARF()
if err != nil {
- fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
+ fatalf("cannot load DWARF output from %s: %v", p.gccTmp(), err)
}
bo := f.ByteOrder
if f.Symtab != nil {
@@ -1795,11 +1803,11 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
return d, ints, floats, strs
}
- if f, err := elf.Open(gccTmp()); err == nil {
+ if f, err := elf.Open(p.gccTmp()); err == nil {
defer f.Close()
d, err := f.DWARF()
if err != nil {
- fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
+ fatalf("cannot load DWARF output from %s: %v", p.gccTmp(), err)
}
bo := f.ByteOrder
symtab, err := f.Symbols()
@@ -1874,11 +1882,11 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
return d, ints, floats, strs
}
- if f, err := pe.Open(gccTmp()); err == nil {
+ if f, err := pe.Open(p.gccTmp()); err == nil {
defer f.Close()
d, err := f.DWARF()
if err != nil {
- fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
+ fatalf("cannot load DWARF output from %s: %v", p.gccTmp(), err)
}
bo := binary.LittleEndian
for _, s := range f.Symbols {
@@ -1946,11 +1954,11 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
return d, ints, floats, strs
}
- if f, err := xcoff.Open(gccTmp()); err == nil {
+ if f, err := xcoff.Open(p.gccTmp()); err == nil {
defer f.Close()
d, err := f.DWARF()
if err != nil {
- fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
+ fatalf("cannot load DWARF output from %s: %v", p.gccTmp(), err)
}
bo := binary.BigEndian
for _, s := range f.Symbols {
@@ -2016,7 +2024,7 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
buildStrings()
return d, ints, floats, strs
}
- fatalf("cannot parse gcc output %s as ELF, Mach-O, PE, XCOFF object", gccTmp())
+ fatalf("cannot parse gcc output %s as ELF, Mach-O, PE, XCOFF object", p.gccTmp())
panic("not reached")
}
@@ -2026,7 +2034,7 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
// and its included files.
func (p *Package) gccDefines(stdin []byte) string {
base := append(gccBaseCmd, "-E", "-dM", "-xc")
- base = append(base, p.gccMachine()...)
+ base = append(base, gccMachine()...)
stdout, _ := runGcc(stdin, append(append(base, p.GccOptions...), "-"))
return stdout
}
@@ -2126,7 +2134,13 @@ type typeConv struct {
intSize int64
}
-var tagGen int
+var tagGenCounter int64
+
+func genTag() int {
+ return int(atomic.AddInt64(&tagGenCounter, 1) - 1)
+}
+
+var globalMu sync.Mutex
var typedef = make(map[string]*Type)
var goIdent = make(map[string]*ast.Ident)
@@ -2134,6 +2148,29 @@ var goIdent = make(map[string]*ast.Ident)
// that may contain a pointer. This is used for cgo pointer checking.
var unionWithPointer = make(map[ast.Expr]bool)
+func unionWithPointerTrueIfKey(setKey, ifKey ast.Expr) {
+ globalMu.Lock()
+ defer globalMu.Unlock()
+
+ if unionWithPointer[ifKey] {
+ unionWithPointer[setKey] = true
+ }
+}
+
+func isUnionWithPointer(key ast.Expr) bool {
+ globalMu.Lock()
+ defer globalMu.Unlock()
+
+ return unionWithPointer[key]
+}
+
+func unionWithPointerTrue(key ast.Expr) {
+ globalMu.Lock()
+ defer globalMu.Unlock()
+
+ unionWithPointer[key] = true
+}
+
// anonymousStructTag provides a consistent tag for an anonymous struct.
// The same dwarf.StructType pointer will always get the same tag.
var anonymousStructTag = make(map[*dwarf.StructType]string)
@@ -2472,9 +2509,7 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
t.Size = t1.Size
t.Align = t1.Align
t.Go = t1.Go
- if unionWithPointer[t1.Go] {
- unionWithPointer[t.Go] = true
- }
+ unionWithPointerTrueIfKey(t.Go, t1.Go)
t.EnumValues = nil
t.Typedef = ""
t.C.Set("%s "+dt.Qual, t1.C)
@@ -2488,18 +2523,21 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
break
}
if tag == "" {
+ globalMu.Lock()
tag = anonymousStructTag[dt]
if tag == "" {
- tag = "__" + strconv.Itoa(tagGen)
- tagGen++
+ tag = "__" + strconv.Itoa(genTag())
anonymousStructTag[dt] = tag
}
+ globalMu.Unlock()
} else if t.C.Empty() {
t.C.Set(dt.Kind + " " + tag)
}
name := c.Ident("_Ctype_" + dt.Kind + "_" + tag)
t.Go = name // publish before recursive calls
+ globalMu.Lock()
goIdent[name.Name] = name
+ globalMu.Unlock()
if dt.ByteSize < 0 {
// Size calculation in c.Struct/c.Opaque will die with size=-1 (unknown),
// so execute the basic things that the struct case would do
@@ -2519,20 +2557,24 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
// on the Go heap, right? It currently doesn't work for unions because
// they are defined as a type alias for struct{}, not a defined type.
}
+ globalMu.Lock()
typedef[name.Name] = &tt
+ globalMu.Unlock()
break
}
switch dt.Kind {
case "class", "union":
t.Go = c.Opaque(t.Size)
if c.dwarfHasPointer(dt, pos) {
- unionWithPointer[t.Go] = true
+ unionWithPointerTrue(t.Go)
}
if t.C.Empty() {
t.C.Set("__typeof__(unsigned char[%d])", t.Size)
}
t.Align = 1 // TODO: should probably base this on field alignment.
+ globalMu.Lock()
typedef[name.Name] = t
+ globalMu.Unlock()
case "struct":
g, csyntax, align := c.Struct(dt, pos)
if t.C.Empty() {
@@ -2545,7 +2587,9 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
}
tt.Go = g
tt.NotInHeap = c.notInHeapStructs[tag]
+ globalMu.Lock()
typedef[name.Name] = &tt
+ globalMu.Unlock()
}
case *dwarf.TypedefType:
@@ -2568,7 +2612,9 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
break
}
name := c.Ident("_Ctype_" + dt.Name)
+ globalMu.Lock()
goIdent[name.Name] = name
+ globalMu.Unlock()
akey := ""
if c.anonymousStructTypedef(dt) {
// only load type recursively for typedefs of anonymous
@@ -2583,10 +2629,12 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
s.BadPointer = true
sub = &s
// Make sure we update any previously computed type.
+ globalMu.Lock()
if oldType := typedef[name.Name]; oldType != nil {
oldType.Go = sub.Go
oldType.BadPointer = true
}
+ globalMu.Unlock()
}
if c.badVoidPointerTypedef(dt) {
// Treat this typedef as a pointer to a NotInHeap void.
@@ -2615,11 +2663,10 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
t.Go = name
t.BadPointer = sub.BadPointer
t.NotInHeap = sub.NotInHeap
- if unionWithPointer[sub.Go] {
- unionWithPointer[t.Go] = true
- }
+ unionWithPointerTrueIfKey(t.Go, sub.Go)
t.Size = sub.Size
t.Align = sub.Align
+ globalMu.Lock()
oldType := typedef[name.Name]
if oldType == nil {
tt := *t
@@ -2628,6 +2675,7 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
tt.NotInHeap = sub.NotInHeap
typedef[name.Name] = &tt
}
+ globalMu.Unlock()
// If sub.Go.Name is "_Ctype_struct_foo" or "_Ctype_union_foo" or "_Ctype_class_foo",
// use that as the Go form for this typedef too, so that the typedef will be interchangeable
@@ -2638,7 +2686,9 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
if isStructUnionClass(sub.Go) {
// Use the typedef name for C code.
+ globalMu.Lock()
typedef[sub.Go.(*ast.Ident).Name].C = t.C
+ globalMu.Unlock()
}
// If we've seen this typedef before, and it
@@ -2698,8 +2748,12 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
}
s = strings.Replace(s, " ", "", -1)
name := c.Ident("_Ctype_" + s)
+
tt := *t
+ globalMu.Lock()
typedef[name.Name] = &tt
+ globalMu.Unlock()
+
if !*godefs {
t.Go = name
}
diff --git a/src/cmd/cgo/godefs.go b/src/cmd/cgo/godefs.go
index c0d59aee01d7..740aa3bc30b2 100644
--- a/src/cmd/cgo/godefs.go
+++ b/src/cmd/cgo/godefs.go
@@ -115,11 +115,11 @@ func (p *Package) godefs(f *File) string {
return buf.String()
}
-var gofmtBuf bytes.Buffer
-
// gofmt returns the gofmt-formatted string for an AST node.
func gofmt(n interface{}) string {
- gofmtBuf.Reset()
+ var gofmtBuf strings.Builder
+ gofmtBuf.Grow(512)
+
err := printer.Fprint(&gofmtBuf, fset, n)
if err != nil {
return "<" + err.Error() + ">"
diff --git a/src/cmd/cgo/main.go b/src/cmd/cgo/main.go
index 14642b7576b0..36e76908c602 100644
--- a/src/cmd/cgo/main.go
+++ b/src/cmd/cgo/main.go
@@ -26,6 +26,7 @@ import (
"runtime"
"sort"
"strings"
+ "sync"
"cmd/internal/edit"
"cmd/internal/objabi"
@@ -33,6 +34,8 @@ import (
// A Package collects information about the package we're going to write.
type Package struct {
+ workerID int
+
PackageName string // name of package
PackagePath string
PtrSize int64
@@ -40,17 +43,36 @@ type Package struct {
GccOptions []string
GccIsClang bool
CgoFlags map[string][]string // #cgo flags (CFLAGS, LDFLAGS)
- Written map[string]bool
- Name map[string]*Name // accumulated Name from Files
+ Written *WrittenFiles
+ Name map[string]*Name // TODO: parallelize. accumulated Name from Files
ExpFunc []*ExpFunc // accumulated ExpFunc from Files
- Decl []ast.Decl
- GoFiles []string // list of Go files
- GccFiles []string // list of gcc output files
- Preamble string // collected preamble for _cgo_export.h
+ Decl []ast.Decl `json:"-"`
+ GoFiles []string // list of Go files
+ GccFiles []string // list of gcc output files
+ Preamble string // collected preamble for _cgo_export.h
+
typedefs map[string]bool // type names that appear in the types of the objects we're interested in
typedefList []typedefInfo
}
+type WrittenFiles struct {
+ mut sync.Mutex
+ written map[string]struct{}
+}
+
+func (w *WrittenFiles) Mark(file string) (shouldWrite bool) {
+ w.mut.Lock()
+ defer w.mut.Unlock()
+
+ _, already := w.written[file]
+ if already {
+ return false
+ }
+
+ w.written[file] = struct{}{}
+ return true
+}
+
// A typedefInfo is an element on Package.typedefList: a typedef name
// and the position where it was required.
type typedefInfo struct {
@@ -87,7 +109,7 @@ func nameKeys(m map[string]*Name) []string {
// A Call refers to a call of a C.xxx function in the AST.
type Call struct {
- Call *ast.CallExpr
+ Call *ast.CallExpr `json:"-"`
Deferred bool
Done bool
}
@@ -95,7 +117,7 @@ type Call struct {
// A Ref refers to an expression of the form C.xxx in the AST.
type Ref struct {
Name *Name
- Expr *ast.Expr
+ Expr *ast.Expr `json:"-"`
Context astContext
Done bool
}
@@ -108,13 +130,13 @@ var nameKinds = []string{"iconst", "fconst", "sconst", "type", "var", "fpvar", "
// A Name collects information about C.xxx.
type Name struct {
- Go string // name used in Go referring to package C
- Mangle string // name used in generated Go
- C string // name used in C
- Define string // #define expansion
- Kind string // one of the nameKinds
- Type *Type // the type of xxx
- FuncType *FuncType
+ Go string // name used in Go referring to package C
+ Mangle string // name used in generated Go
+ C string // name used in C
+ Define string // #define expansion
+ Kind string // one of the nameKinds
+ Type *Type `json:"-"` // the type of xxx
+ FuncType *FuncType `json:"-"`
AddError bool
Const string // constant definition
}
@@ -133,23 +155,23 @@ func (n *Name) IsConst() bool {
// Such functions are identified in the Go input file
// by doc comments containing the line //export ExpName
type ExpFunc struct {
- Func *ast.FuncDecl
- ExpName string // name to use from C
+ Func *ast.FuncDecl `json:"-"`
+ ExpName string // name to use from C
Doc string
}
// A TypeRepr contains the string representation of a type.
type TypeRepr struct {
Repr string
- FormatArgs []interface{}
+ FormatArgs []interface{} `json:"-"`
}
// A Type collects information about a type in both the C and Go worlds.
type Type struct {
Size int64
Align int64
- C *TypeRepr
- Go ast.Expr
+ C *TypeRepr `json:"-"`
+ Go ast.Expr `json:"-"`
EnumValues map[string]int64
Typedef string
BadPointer bool // this pointer type should be represented as a uintptr (deprecated)
@@ -223,6 +245,8 @@ var cPrefix string
var fset = token.NewFileSet()
+var nparallel = flag.Int("nparallel", runtime.GOMAXPROCS(0), "maximum parallel jobs")
+
var dynobj = flag.String("dynimport", "", "if non-empty, print dynamic import data for that file")
var dynout = flag.String("dynout", "", "write -dynimport output to this file")
var dynpackage = flag.String("dynpackage", "main", "set Go package for -dynimport output")
@@ -241,11 +265,13 @@ var exportHeader = flag.String("exportheader", "", "where to write export header
var gccgo = flag.Bool("gccgo", false, "generate files for use with gccgo")
var gccgoprefix = flag.String("gccgoprefix", "", "-fgo-prefix option used with gccgo")
var gccgopkgpath = flag.String("gccgopkgpath", "", "-fgo-pkgpath option used with gccgo")
-var gccgoMangler func(string) string
var importRuntimeCgo = flag.Bool("import_runtime_cgo", true, "import runtime/cgo in generated code")
var importSyscall = flag.Bool("import_syscall", true, "import syscall in generated code")
var trimpath = flag.String("trimpath", "", "applies supplied rewrites or trims prefixes to recorded source file paths")
+var gccgoMangler func(string) string
+var gccgoManglerOnce sync.Once
+
var goarch, goos, gomips, gomips64 string
var gccBaseCmd []string
@@ -302,7 +328,7 @@ func main() {
}
}
- p := newPackage(args[:i])
+ p := newPackage(args[:i], 0)
// We need a C compiler to be available. Check this.
var err error
@@ -370,43 +396,152 @@ func main() {
}
*objDir += string(filepath.Separator)
- for i, input := range goFiles {
- f := fs[i]
- p.Translate(f)
- for _, cref := range f.Ref {
- switch cref.Context {
- case ctxCall, ctxCall2:
- if cref.Name.Kind != "type" {
- break
- }
- old := *cref.Expr
- *cref.Expr = cref.Name.Type.Go
- f.Edit.Replace(f.offset(old.Pos()), f.offset(old.End()), gofmt(cref.Name.Type.Go))
- }
- }
- if nerrors > 0 {
- os.Exit(2)
- }
- p.PackagePath = f.Package
- p.Record(f)
- if *godefs {
- os.Stdout.WriteString(p.godefs(f))
- } else {
- p.writeOutput(f, input)
+ pkgs := doFiles(p, goFiles, fs)
+
+ if !*godefs {
+ pkg := coalescePackages(pkgs)
+ pkg.writeDefs()
+ }
+
+ if nerrors.get() > 0 {
+ os.Exit(2)
+ }
+}
+
+type doFileJob struct {
+ input string
+ file *File
+}
+
+func doFiles(p *Package, inputs []string, files []*File) []*Package {
+ if len(inputs) != len(files) {
+ fatalf("unexpected len(inputs) %d != len(files) %d", len(inputs), len(files))
+ }
+
+ jobCh := make(chan doFileJob)
+
+ var wg sync.WaitGroup
+ wg.Add(*nparallel)
+
+ pkgs := make([]*Package, *nparallel)
+ pkgs[0] = p
+
+ for n := 1; n < *nparallel; n++ {
+ pkgs[n] = cloneNewPackage(p, n)
+ }
+
+ for n := 0; n < *nparallel; n++ {
+ p := pkgs[n]
+ go func() {
+ doFileWorker(p, jobCh)
+ wg.Done()
+ }()
+ }
+
+ for i := range inputs {
+ jobCh <- doFileJob{
+ input: inputs[i],
+ file: files[i],
}
}
- if !*godefs {
- p.writeDefs()
+ close(jobCh)
+ wg.Wait()
+
+ return pkgs
+}
+
+func doFileWorker(p *Package, jobCh <-chan doFileJob) *Package {
+ for job := range jobCh {
+ doFile(p, job.input, job.file)
}
- if nerrors > 0 {
+
+ return p
+}
+
+func doFile(p *Package, input string, f *File) {
+ p.Translate(f)
+ for _, cref := range f.Ref {
+ switch cref.Context {
+ case ctxCall, ctxCall2:
+ if cref.Name.Kind != "type" {
+ break
+ }
+ old := *cref.Expr
+ *cref.Expr = cref.Name.Type.Go
+ f.Edit.Replace(f.offset(old.Pos()), f.offset(old.End()), gofmt(cref.Name.Type.Go))
+ }
+ }
+ if nerrors.get() > 0 {
os.Exit(2)
}
+ p.PackagePath = f.Package
+ p.Record(f)
+ if *godefs {
+ os.Stdout.WriteString(p.godefs(f))
+ } else {
+ p.writeOutput(f, input)
+ }
+}
+
+func cloneNewPackage(p *Package, workerID int) *Package {
+ return &Package{
+ workerID: workerID,
+ PackageName: p.PackageName,
+ PackagePath: p.PackagePath,
+ PtrSize: p.PtrSize,
+ IntSize: p.IntSize,
+ GccOptions: p.GccOptions, // init in main only
+ GccIsClang: p.GccIsClang,
+ CgoFlags: p.CgoFlags, // init in main only
+ Written: p.Written,
+ }
+}
+
+func coalescePackages(pkgs []*Package) *Package {
+ if len(pkgs) == 0 {
+ return nil
+ }
+
+ nonEmpty := pkgs[:0]
+ for _, p := range pkgs {
+ if p.PackageName != "" {
+ nonEmpty = append(nonEmpty, p)
+ }
+ }
+
+ pkgs = nonEmpty
+ pkg := pkgs[0]
+
+ for _, p := range pkgs[1:] {
+ pkg.PackageName = p.PackageName
+ pkg.PackagePath = p.PackagePath
+ pkg.ExpFunc = append(pkg.ExpFunc, p.ExpFunc...)
+ pkg.Decl = append(pkg.Decl, p.Decl...)
+ pkg.GoFiles = append(pkg.GoFiles, p.GoFiles...)
+ pkg.GccFiles = append(pkg.GccFiles, p.GccFiles...)
+ pkg.Preamble += p.Preamble
+
+ if pkg.Name == nil && p.Name != nil {
+ pkg.Name = p.Name
+ continue
+ }
+
+ for k, v := range p.Name {
+ if o, ok := pkg.Name[k]; ok && !reflect.DeepEqual(v, o) {
+ fatalf("duplicate pkg.Name %s, %v == %v", k, v, o)
+ }
+
+ pkg.Name[k] = v
+ }
+ }
+
+ return pkg
}
// newPackage returns a new Package that will invoke
// gcc with the additional arguments specified in args.
-func newPackage(args []string) *Package {
+func newPackage(args []string, workerID int) *Package {
goarch = runtime.GOARCH
if s := os.Getenv("GOARCH"); s != "" {
goarch = s
@@ -432,10 +567,13 @@ func newPackage(args []string) *Package {
os.Setenv("LC_ALL", "C")
p := &Package{
+ workerID: workerID,
PtrSize: ptrSize,
IntSize: intSize,
CgoFlags: make(map[string][]string),
- Written: make(map[string]bool),
+ Written: &WrittenFiles{
+ written: make(map[string]struct{}),
+ },
}
p.addToFlag("CFLAGS", args)
return p
diff --git a/src/cmd/cgo/out.go b/src/cmd/cgo/out.go
index 4968f7059d9b..b934d12d9423 100644
--- a/src/cmd/cgo/out.go
+++ b/src/cmd/cgo/out.go
@@ -32,14 +32,13 @@ var (
// writeDefs creates output files to be compiled by gc and gcc.
func (p *Package) writeDefs() {
- var fgo2, fc io.Writer
- f := creat(*objDir + "_cgo_gotypes.go")
- defer f.Close()
- fgo2 = f
+ fgo2 := creat(*objDir + "_cgo_gotypes.go")
+ defer fgo2.Close()
+
+ var fc *os.File
if *gccgo {
- f := creat(*objDir + "_cgo_defun.c")
- defer f.Close()
- fc = f
+ fc = creat(*objDir + "_cgo_defun.c")
+ defer fc.Close()
}
fm := creat(*objDir + "_cgo_main.c")
@@ -682,12 +681,11 @@ var isBuiltin = map[string]bool{
func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) {
name := n.Mangle
- if isBuiltin[name] || p.Written[name] {
+ if isBuiltin[name] || !p.Written.Mark(name) {
// The builtins are already defined in the C prolog, and we don't
// want to duplicate function definitions we've already done.
return
}
- p.Written[name] = true
if *gccgo {
p.writeGccgoOutputFunc(fgcc, n)
@@ -1294,7 +1292,7 @@ func (p *Package) writeExportHeader(fgcch io.Writer) {
// gccgoToSymbol converts a name to a mangled symbol for gccgo.
func gccgoToSymbol(ppath string) string {
- if gccgoMangler == nil {
+ gccgoManglerOnce.Do(func() {
var err error
cmd := os.Getenv("GCCGO")
if cmd == "" {
@@ -1307,7 +1305,7 @@ func gccgoToSymbol(ppath string) string {
if err != nil {
fatalf("%v", err)
}
- }
+ })
return gccgoMangler(ppath)
}
diff --git a/src/cmd/cgo/util.go b/src/cmd/cgo/util.go
index 00d931b98a0c..bea77b841466 100644
--- a/src/cmd/cgo/util.go
+++ b/src/cmd/cgo/util.go
@@ -11,6 +11,7 @@ import (
exec "internal/execabs"
"io/ioutil"
"os"
+ "sync/atomic"
)
// run runs the command argv, feeding in stdin on standard input.
@@ -87,16 +88,22 @@ func lineno(pos token.Pos) string {
func fatalf(msg string, args ...interface{}) {
// If we've already printed other errors, they might have
// caused the fatal condition. Assume they're enough.
- if nerrors == 0 {
+ if !nerrors.has() {
fmt.Fprintf(os.Stderr, "cgo: "+msg+"\n", args...)
}
os.Exit(2)
}
-var nerrors int
+type atomicNErrors uint64
+
+var nerrors atomicNErrors
+
+func (n *atomicNErrors) add() { atomic.AddUint64((*uint64)(n), 1) }
+func (n *atomicNErrors) has() bool { return n.get() > 0 }
+func (n *atomicNErrors) get() uint64 { return atomic.LoadUint64((*uint64)(n)) }
func error_(pos token.Pos, msg string, args ...interface{}) {
- nerrors++
+ nerrors.add()
if pos.IsValid() {
fmt.Fprintf(os.Stderr, "%s: ", fset.Position(pos).String())
} else {
diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go
index 0af39a991c02..ac658f4803ef 100644
--- a/src/cmd/go/internal/work/exec.go
+++ b/src/cmd/go/internal/work/exec.go
@@ -2821,7 +2821,12 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo
cgoflags = append(cgoflags, "-trimpath", strings.Join(trimpath, ";"))
}
- if err := b.run(a, execdir, p.ImportPath, cgoenv, cfg.BuildToolexec, cgoExe, "-objdir", objdir, "-importpath", p.ImportPath, cgoflags, "--", cgoCPPFLAGS, cgoCFLAGS, cgofiles); err != nil {
+ nparallel := runtime.GOMAXPROCS(0)
+ if nparallel > 4 {
+ nparallel = 4
+ }
+
+ if err := b.run(a, execdir, p.ImportPath, cgoenv, cfg.BuildToolexec, cgoExe, "-nparallel", strconv.Itoa(nparallel), "-objdir", objdir, "-importpath", p.ImportPath, cgoflags, "--", cgoCPPFLAGS, cgoCFLAGS, cgofiles); err != nil {
return nil, nil, err
}
outGo = append(outGo, gofiles...)
Reference in New Issue Copy Permalink