flake,go: add,use overlay.nix w/ patched Go
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
This commit is contained in:
parent
f513c92159
commit
fb0d215760
29
flake.nix
29
flake.nix
@ -31,13 +31,18 @@
|
||||
forAllSystems = nixpkgs.lib.genAttrs supportedSystems;
|
||||
|
||||
# Nixpkgs instantiated for supported system types.
|
||||
nixpkgsFor = forAllSystems (system: import nixpkgs { inherit system; overlays = [ ]; });
|
||||
nixpkgsFor = forAllSystems (system: import nixpkgs {
|
||||
inherit system;
|
||||
overlays = [
|
||||
(import ./overlay.nix)
|
||||
];
|
||||
});
|
||||
in
|
||||
rec {
|
||||
packages = forAllSystems (system:
|
||||
let
|
||||
pkgs = nixpkgsFor.${system};
|
||||
inherit (pkgs) lib;# -> lib = pkgs.lib;
|
||||
inherit (pkgs) lib overlays;# -> lib = pkgs.lib;overlays = pkgs.overlays;
|
||||
in
|
||||
rec {
|
||||
go-xkcdreader = with pkgs; buildGoModule rec {
|
||||
@ -73,8 +78,8 @@
|
||||
];
|
||||
|
||||
modSha256 = lib.fakeSha256;
|
||||
# vendorSha256 = "";
|
||||
vendorSha256 = "sha256-i6VOX1O/IqIOWUzgc/2U/a95KhzAG0/NOIG/D8E0BEk=";
|
||||
# dont't forget to update vendorSha256 whenever go.mod or go.sum change
|
||||
vendorSha256 = "sha256-oHOMkvQhMFsAGgMcAHvxZp1vcDSVLmUYhft+cvnMd6M=";
|
||||
|
||||
# In 'nix develop', we don't need a copy of the source tree
|
||||
# in the Nix store.
|
||||
@ -107,6 +112,13 @@
|
||||
inherit system;
|
||||
overlays = [ nixgl.overlay ];
|
||||
};
|
||||
goPkgs = import nixpkgs {
|
||||
useFetched = true;
|
||||
inherit system;
|
||||
overlays = self.overlays or [ ] ++ [
|
||||
(import ./overlay.nix)
|
||||
];
|
||||
};
|
||||
in
|
||||
{
|
||||
default = with pkgs; mkShell
|
||||
@ -132,7 +144,14 @@
|
||||
# pkgs.nixgl.auto.nixGLDefault # requires --impure
|
||||
];
|
||||
packages = [
|
||||
go
|
||||
## use patched Go, since it's supposed to be faster
|
||||
goPkgs.go
|
||||
## if you wish to use this, uncomment the related block in
|
||||
## overlay.nix and the next line
|
||||
# goPkgs.dominikh.go-tools
|
||||
goPkgs.patchelf-x86_64
|
||||
goPkgs.patchelf-aarch64
|
||||
|
||||
gopls
|
||||
gofumpt
|
||||
go-tools
|
||||
|
63
overlay.nix
Normal file
63
overlay.nix
Normal file
@ -0,0 +1,63 @@
|
||||
# heavily inspired by https://github.com/diamondburned/gotk4-nix/blob/d2bd6577f1867cb740b281baa48a895aed494967/overlay.nix
|
||||
self: super:
|
||||
let patchelfer = arch: interpreter: super.writeShellScriptBin
|
||||
"patchelf-${arch}"
|
||||
"${super.patchelf}/bin/patchelf --set-interpreter ${interpreter} \"$@\"";
|
||||
in
|
||||
{
|
||||
go = super.go.overrideAttrs (old: {
|
||||
version = "1.18.2";
|
||||
src = builtins.fetchurl {
|
||||
url = "https://go.dev/dl/go1.18.2.src.tar.gz";
|
||||
sha256 = "sha256:1qk7as7a5wx3522ldsq8il1il9ic5xhbl6drg89r4h63l8zd0i1c";
|
||||
};
|
||||
doCheck = false;
|
||||
patches = (old.patches or [ ]) ++ [
|
||||
# cmd/go/internal/work: concurrent ccompile routines
|
||||
(super.fetchpatch {
|
||||
url = "https://github.com/diamondburned/go/commit/ec3e1c9471f170187b6a7c83ab0364253f895c28.patch";
|
||||
sha256 = "sha256-KXP3gHRVHCVY60HnrLHv+QLib1hSYZOxsMjkbAiNi1g=";
|
||||
})
|
||||
# cmd/cgo: concurrent file generation
|
||||
(super.fetchpatch {
|
||||
url = "https://github.com/diamondburned/go/commit/50e04befeca9ae63296a73c8d5d2870b904971b4.patch";
|
||||
sha256 = "sha256-6PNUBlhGj53XS7f3dYOisDQ1PJldqwmWcgBnPE0cZsE=";
|
||||
})
|
||||
# the patches are also present locally for archival purposes
|
||||
# ./patches/diamondburned/go/ec3e1c9471f170187b6a7c83ab0364253f895c28.patch
|
||||
# ./patches/diamondburned/go/50e04befeca9ae63296a73c8d5d2870b904971b4.patch
|
||||
# TODO(me): don't forget to download the patch files when bumping the
|
||||
# urls above
|
||||
];
|
||||
});
|
||||
|
||||
# might as well be 'buildGoModule = super.buildGo118Module.override' but
|
||||
# we're overriding just the go package itself and leaving the rest of the
|
||||
# function alone. letting 'builGoModule' to be updated only with flake's
|
||||
# nixpkgs updates therefore seems to be the more reasonable option.
|
||||
buildGoModule = super.buildGoModule.override {
|
||||
# use the go we just built from sources as 'go' in the buildGoModule func
|
||||
inherit (self) go;
|
||||
};
|
||||
|
||||
# inherit (super) go-tools;
|
||||
|
||||
# See https://sourceware.org/glibc/wiki/ABIList.
|
||||
patchelf-x86_64 = patchelfer "x86_64" "/lib64/ld-linux-x86-64.so.2";
|
||||
patchelf-aarch64 = patchelfer "aarch64" "/lib/ld-linux-aarch64.so.1";
|
||||
|
||||
#dominikh = {
|
||||
# go-tools = self.buildGoModule {
|
||||
# name = "dominikh-go-tools";
|
||||
# src = super.fetchFromGitHub {
|
||||
# owner = "dominikh";
|
||||
# repo = "go-tools";
|
||||
# rev = "f4a2f64ce2386d1d392f2da44043c5ee3fb48216";
|
||||
# sha256 = "1yhbz2sf332b6i00slsj4cn8r66x27kddw5vcjygkkiyny1a99qb";
|
||||
# };
|
||||
# vendorSha256 = "09jbarlbq47pcxy5zkja8gqvnqjp2mpbxnciv9lhilw9swqqwc0j";
|
||||
# doCheck = false;
|
||||
# subPackages = [ "cmd/staticcheck" ];
|
||||
# };
|
||||
#};
|
||||
}
|
@ -0,0 +1,883 @@
|
||||
From 50e04befeca9ae63296a73c8d5d2870b904971b4 Mon Sep 17 00:00:00 2001
|
||||
From: diamondburned <datutbrus@gmail.com>
|
||||
Date: Thu, 5 Aug 2021 16:47:49 -0700
|
||||
Subject: [PATCH] cmd/cgo: concurrent file generation
|
||||
|
||||
This commit allows cmd/cgo to generate files in parallel to each other.
|
||||
The parallelism is determined by either the thread count (using
|
||||
$GOMAXPROCS) or the -nparallel flag, which is set in
|
||||
cmd/go/internal/work to be maximum 4.
|
||||
|
||||
The commit addresses some existing race conditions that had to do with
|
||||
mutating the global state by using a global write mutex during the
|
||||
translation stage.
|
||||
|
||||
Other race conditions that had to do with mutating the Package singleton
|
||||
was solved by giving each translation worker its own package, and then
|
||||
combining them back into a single package.
|
||||
|
||||
With this and the cmd/go/internal/work changes in place, build time
|
||||
(from scratch, with -a) for gotktrix is reduced from 33 minutes to 10
|
||||
minutes on an 8-thread Intel Core i5-8250U.
|
||||
---
|
||||
src/cmd/cgo/gcc.go | 120 +++++++++++-----
|
||||
src/cmd/cgo/godefs.go | 6 +-
|
||||
src/cmd/cgo/main.go | 238 ++++++++++++++++++++++++-------
|
||||
src/cmd/cgo/out.go | 20 ++-
|
||||
src/cmd/cgo/util.go | 13 +-
|
||||
src/cmd/go/internal/work/exec.go | 7 +-
|
||||
6 files changed, 303 insertions(+), 101 deletions(-)
|
||||
|
||||
diff --git a/src/cmd/cgo/gcc.go b/src/cmd/cgo/gcc.go
|
||||
index 997a830994f0..59a8405c0c71 100644
|
||||
--- a/src/cmd/cgo/gcc.go
|
||||
+++ b/src/cmd/cgo/gcc.go
|
||||
@@ -26,6 +26,8 @@ import (
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
+ "sync"
|
||||
+ "sync/atomic"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
@@ -485,7 +487,7 @@ func (p *Package) guessKinds(f *File) []*Name {
|
||||
}
|
||||
needType = append(needType, n)
|
||||
}
|
||||
- if nerrors > 0 {
|
||||
+ if nerrors.has() {
|
||||
// Check if compiling the preamble by itself causes any errors,
|
||||
// because the messages we've printed out so far aren't helpful
|
||||
// to users debugging preamble mistakes. See issue 8442.
|
||||
@@ -727,9 +729,12 @@ func (p *Package) prepareNames(f *File) {
|
||||
}
|
||||
}
|
||||
p.mangleName(n)
|
||||
+
|
||||
+ globalMu.Lock()
|
||||
if n.Kind == "type" && typedef[n.Mangle] == nil {
|
||||
typedef[n.Mangle] = n.Type
|
||||
}
|
||||
+ globalMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1023,7 +1028,7 @@ func (p *Package) hasPointer(f *File, t ast.Expr, top bool) bool {
|
||||
}
|
||||
// Check whether this is a pointer to a C union (or class)
|
||||
// type that contains a pointer.
|
||||
- if unionWithPointer[t.X] {
|
||||
+ if isUnionWithPointer(t.X) {
|
||||
return true
|
||||
}
|
||||
return p.hasPointer(f, t.X, false)
|
||||
@@ -1046,7 +1051,10 @@ func (p *Package) hasPointer(f *File, t ast.Expr, top bool) bool {
|
||||
}
|
||||
}
|
||||
}
|
||||
- if def := typedef[t.Name]; def != nil {
|
||||
+ globalMu.Lock()
|
||||
+ def := typedef[t.Name]
|
||||
+ globalMu.Unlock()
|
||||
+ if def != nil {
|
||||
return p.hasPointer(f, def.Go, top)
|
||||
}
|
||||
if t.Name == "string" {
|
||||
@@ -1582,7 +1590,7 @@ func checkGCCBaseCmd() ([]string, error) {
|
||||
}
|
||||
|
||||
// gccMachine returns the gcc -m flag to use, either "-m32", "-m64" or "-marm".
|
||||
-func (p *Package) gccMachine() []string {
|
||||
+func gccMachine() []string {
|
||||
switch goarch {
|
||||
case "amd64":
|
||||
if goos == "darwin" {
|
||||
@@ -1617,20 +1625,20 @@ func (p *Package) gccMachine() []string {
|
||||
return nil
|
||||
}
|
||||
|
||||
-func gccTmp() string {
|
||||
- return *objDir + "_cgo_.o"
|
||||
+func (p *Package) gccTmp() string {
|
||||
+ return fmt.Sprintf("%s_%d_cgo_.o", *objDir, p.workerID)
|
||||
}
|
||||
|
||||
// gccCmd returns the gcc command line to use for compiling
|
||||
// the input.
|
||||
func (p *Package) gccCmd() []string {
|
||||
c := append(gccBaseCmd,
|
||||
- "-w", // no warnings
|
||||
- "-Wno-error", // warnings are not errors
|
||||
- "-o"+gccTmp(), // write object to tmp
|
||||
- "-gdwarf-2", // generate DWARF v2 debugging symbols
|
||||
- "-c", // do not link
|
||||
- "-xc", // input language is C
|
||||
+ "-w", // no warnings
|
||||
+ "-Wno-error", // warnings are not errors
|
||||
+ "-o"+p.gccTmp(), // write object to tmp
|
||||
+ "-gdwarf-2", // generate DWARF v2 debugging symbols
|
||||
+ "-c", // do not link
|
||||
+ "-xc", // input language is C
|
||||
)
|
||||
if p.GccIsClang {
|
||||
c = append(c,
|
||||
@@ -1653,7 +1661,7 @@ func (p *Package) gccCmd() []string {
|
||||
}
|
||||
|
||||
c = append(c, p.GccOptions...)
|
||||
- c = append(c, p.gccMachine()...)
|
||||
+ c = append(c, gccMachine()...)
|
||||
if goos == "aix" {
|
||||
c = append(c, "-maix64")
|
||||
c = append(c, "-mcmodel=large")
|
||||
@@ -1717,11 +1725,11 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
|
||||
}
|
||||
}
|
||||
|
||||
- if f, err := macho.Open(gccTmp()); err == nil {
|
||||
+ if f, err := macho.Open(p.gccTmp()); err == nil {
|
||||
defer f.Close()
|
||||
d, err := f.DWARF()
|
||||
if err != nil {
|
||||
- fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
|
||||
+ fatalf("cannot load DWARF output from %s: %v", p.gccTmp(), err)
|
||||
}
|
||||
bo := f.ByteOrder
|
||||
if f.Symtab != nil {
|
||||
@@ -1795,11 +1803,11 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
|
||||
return d, ints, floats, strs
|
||||
}
|
||||
|
||||
- if f, err := elf.Open(gccTmp()); err == nil {
|
||||
+ if f, err := elf.Open(p.gccTmp()); err == nil {
|
||||
defer f.Close()
|
||||
d, err := f.DWARF()
|
||||
if err != nil {
|
||||
- fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
|
||||
+ fatalf("cannot load DWARF output from %s: %v", p.gccTmp(), err)
|
||||
}
|
||||
bo := f.ByteOrder
|
||||
symtab, err := f.Symbols()
|
||||
@@ -1874,11 +1882,11 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
|
||||
return d, ints, floats, strs
|
||||
}
|
||||
|
||||
- if f, err := pe.Open(gccTmp()); err == nil {
|
||||
+ if f, err := pe.Open(p.gccTmp()); err == nil {
|
||||
defer f.Close()
|
||||
d, err := f.DWARF()
|
||||
if err != nil {
|
||||
- fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
|
||||
+ fatalf("cannot load DWARF output from %s: %v", p.gccTmp(), err)
|
||||
}
|
||||
bo := binary.LittleEndian
|
||||
for _, s := range f.Symbols {
|
||||
@@ -1946,11 +1954,11 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
|
||||
return d, ints, floats, strs
|
||||
}
|
||||
|
||||
- if f, err := xcoff.Open(gccTmp()); err == nil {
|
||||
+ if f, err := xcoff.Open(p.gccTmp()); err == nil {
|
||||
defer f.Close()
|
||||
d, err := f.DWARF()
|
||||
if err != nil {
|
||||
- fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
|
||||
+ fatalf("cannot load DWARF output from %s: %v", p.gccTmp(), err)
|
||||
}
|
||||
bo := binary.BigEndian
|
||||
for _, s := range f.Symbols {
|
||||
@@ -2016,7 +2024,7 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
|
||||
buildStrings()
|
||||
return d, ints, floats, strs
|
||||
}
|
||||
- fatalf("cannot parse gcc output %s as ELF, Mach-O, PE, XCOFF object", gccTmp())
|
||||
+ fatalf("cannot parse gcc output %s as ELF, Mach-O, PE, XCOFF object", p.gccTmp())
|
||||
panic("not reached")
|
||||
}
|
||||
|
||||
@@ -2026,7 +2034,7 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
|
||||
// and its included files.
|
||||
func (p *Package) gccDefines(stdin []byte) string {
|
||||
base := append(gccBaseCmd, "-E", "-dM", "-xc")
|
||||
- base = append(base, p.gccMachine()...)
|
||||
+ base = append(base, gccMachine()...)
|
||||
stdout, _ := runGcc(stdin, append(append(base, p.GccOptions...), "-"))
|
||||
return stdout
|
||||
}
|
||||
@@ -2126,7 +2134,13 @@ type typeConv struct {
|
||||
intSize int64
|
||||
}
|
||||
|
||||
-var tagGen int
|
||||
+var tagGenCounter int64
|
||||
+
|
||||
+func genTag() int {
|
||||
+ return int(atomic.AddInt64(&tagGenCounter, 1) - 1)
|
||||
+}
|
||||
+
|
||||
+var globalMu sync.Mutex
|
||||
var typedef = make(map[string]*Type)
|
||||
var goIdent = make(map[string]*ast.Ident)
|
||||
|
||||
@@ -2134,6 +2148,29 @@ var goIdent = make(map[string]*ast.Ident)
|
||||
// that may contain a pointer. This is used for cgo pointer checking.
|
||||
var unionWithPointer = make(map[ast.Expr]bool)
|
||||
|
||||
+func unionWithPointerTrueIfKey(setKey, ifKey ast.Expr) {
|
||||
+ globalMu.Lock()
|
||||
+ defer globalMu.Unlock()
|
||||
+
|
||||
+ if unionWithPointer[ifKey] {
|
||||
+ unionWithPointer[setKey] = true
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func isUnionWithPointer(key ast.Expr) bool {
|
||||
+ globalMu.Lock()
|
||||
+ defer globalMu.Unlock()
|
||||
+
|
||||
+ return unionWithPointer[key]
|
||||
+}
|
||||
+
|
||||
+func unionWithPointerTrue(key ast.Expr) {
|
||||
+ globalMu.Lock()
|
||||
+ defer globalMu.Unlock()
|
||||
+
|
||||
+ unionWithPointer[key] = true
|
||||
+}
|
||||
+
|
||||
// anonymousStructTag provides a consistent tag for an anonymous struct.
|
||||
// The same dwarf.StructType pointer will always get the same tag.
|
||||
var anonymousStructTag = make(map[*dwarf.StructType]string)
|
||||
@@ -2472,9 +2509,7 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
|
||||
t.Size = t1.Size
|
||||
t.Align = t1.Align
|
||||
t.Go = t1.Go
|
||||
- if unionWithPointer[t1.Go] {
|
||||
- unionWithPointer[t.Go] = true
|
||||
- }
|
||||
+ unionWithPointerTrueIfKey(t.Go, t1.Go)
|
||||
t.EnumValues = nil
|
||||
t.Typedef = ""
|
||||
t.C.Set("%s "+dt.Qual, t1.C)
|
||||
@@ -2488,18 +2523,21 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
|
||||
break
|
||||
}
|
||||
if tag == "" {
|
||||
+ globalMu.Lock()
|
||||
tag = anonymousStructTag[dt]
|
||||
if tag == "" {
|
||||
- tag = "__" + strconv.Itoa(tagGen)
|
||||
- tagGen++
|
||||
+ tag = "__" + strconv.Itoa(genTag())
|
||||
anonymousStructTag[dt] = tag
|
||||
}
|
||||
+ globalMu.Unlock()
|
||||
} else if t.C.Empty() {
|
||||
t.C.Set(dt.Kind + " " + tag)
|
||||
}
|
||||
name := c.Ident("_Ctype_" + dt.Kind + "_" + tag)
|
||||
t.Go = name // publish before recursive calls
|
||||
+ globalMu.Lock()
|
||||
goIdent[name.Name] = name
|
||||
+ globalMu.Unlock()
|
||||
if dt.ByteSize < 0 {
|
||||
// Size calculation in c.Struct/c.Opaque will die with size=-1 (unknown),
|
||||
// so execute the basic things that the struct case would do
|
||||
@@ -2519,20 +2557,24 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
|
||||
// on the Go heap, right? It currently doesn't work for unions because
|
||||
// they are defined as a type alias for struct{}, not a defined type.
|
||||
}
|
||||
+ globalMu.Lock()
|
||||
typedef[name.Name] = &tt
|
||||
+ globalMu.Unlock()
|
||||
break
|
||||
}
|
||||
switch dt.Kind {
|
||||
case "class", "union":
|
||||
t.Go = c.Opaque(t.Size)
|
||||
if c.dwarfHasPointer(dt, pos) {
|
||||
- unionWithPointer[t.Go] = true
|
||||
+ unionWithPointerTrue(t.Go)
|
||||
}
|
||||
if t.C.Empty() {
|
||||
t.C.Set("__typeof__(unsigned char[%d])", t.Size)
|
||||
}
|
||||
t.Align = 1 // TODO: should probably base this on field alignment.
|
||||
+ globalMu.Lock()
|
||||
typedef[name.Name] = t
|
||||
+ globalMu.Unlock()
|
||||
case "struct":
|
||||
g, csyntax, align := c.Struct(dt, pos)
|
||||
if t.C.Empty() {
|
||||
@@ -2545,7 +2587,9 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
|
||||
}
|
||||
tt.Go = g
|
||||
tt.NotInHeap = c.notInHeapStructs[tag]
|
||||
+ globalMu.Lock()
|
||||
typedef[name.Name] = &tt
|
||||
+ globalMu.Unlock()
|
||||
}
|
||||
|
||||
case *dwarf.TypedefType:
|
||||
@@ -2568,7 +2612,9 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
|
||||
break
|
||||
}
|
||||
name := c.Ident("_Ctype_" + dt.Name)
|
||||
+ globalMu.Lock()
|
||||
goIdent[name.Name] = name
|
||||
+ globalMu.Unlock()
|
||||
akey := ""
|
||||
if c.anonymousStructTypedef(dt) {
|
||||
// only load type recursively for typedefs of anonymous
|
||||
@@ -2583,10 +2629,12 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
|
||||
s.BadPointer = true
|
||||
sub = &s
|
||||
// Make sure we update any previously computed type.
|
||||
+ globalMu.Lock()
|
||||
if oldType := typedef[name.Name]; oldType != nil {
|
||||
oldType.Go = sub.Go
|
||||
oldType.BadPointer = true
|
||||
}
|
||||
+ globalMu.Unlock()
|
||||
}
|
||||
if c.badVoidPointerTypedef(dt) {
|
||||
// Treat this typedef as a pointer to a NotInHeap void.
|
||||
@@ -2615,11 +2663,10 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
|
||||
t.Go = name
|
||||
t.BadPointer = sub.BadPointer
|
||||
t.NotInHeap = sub.NotInHeap
|
||||
- if unionWithPointer[sub.Go] {
|
||||
- unionWithPointer[t.Go] = true
|
||||
- }
|
||||
+ unionWithPointerTrueIfKey(t.Go, sub.Go)
|
||||
t.Size = sub.Size
|
||||
t.Align = sub.Align
|
||||
+ globalMu.Lock()
|
||||
oldType := typedef[name.Name]
|
||||
if oldType == nil {
|
||||
tt := *t
|
||||
@@ -2628,6 +2675,7 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
|
||||
tt.NotInHeap = sub.NotInHeap
|
||||
typedef[name.Name] = &tt
|
||||
}
|
||||
+ globalMu.Unlock()
|
||||
|
||||
// If sub.Go.Name is "_Ctype_struct_foo" or "_Ctype_union_foo" or "_Ctype_class_foo",
|
||||
// use that as the Go form for this typedef too, so that the typedef will be interchangeable
|
||||
@@ -2638,7 +2686,9 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
|
||||
|
||||
if isStructUnionClass(sub.Go) {
|
||||
// Use the typedef name for C code.
|
||||
+ globalMu.Lock()
|
||||
typedef[sub.Go.(*ast.Ident).Name].C = t.C
|
||||
+ globalMu.Unlock()
|
||||
}
|
||||
|
||||
// If we've seen this typedef before, and it
|
||||
@@ -2698,8 +2748,12 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
|
||||
}
|
||||
s = strings.Replace(s, " ", "", -1)
|
||||
name := c.Ident("_Ctype_" + s)
|
||||
+
|
||||
tt := *t
|
||||
+ globalMu.Lock()
|
||||
typedef[name.Name] = &tt
|
||||
+ globalMu.Unlock()
|
||||
+
|
||||
if !*godefs {
|
||||
t.Go = name
|
||||
}
|
||||
diff --git a/src/cmd/cgo/godefs.go b/src/cmd/cgo/godefs.go
|
||||
index c0d59aee01d7..740aa3bc30b2 100644
|
||||
--- a/src/cmd/cgo/godefs.go
|
||||
+++ b/src/cmd/cgo/godefs.go
|
||||
@@ -115,11 +115,11 @@ func (p *Package) godefs(f *File) string {
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
-var gofmtBuf bytes.Buffer
|
||||
-
|
||||
// gofmt returns the gofmt-formatted string for an AST node.
|
||||
func gofmt(n interface{}) string {
|
||||
- gofmtBuf.Reset()
|
||||
+ var gofmtBuf strings.Builder
|
||||
+ gofmtBuf.Grow(512)
|
||||
+
|
||||
err := printer.Fprint(&gofmtBuf, fset, n)
|
||||
if err != nil {
|
||||
return "<" + err.Error() + ">"
|
||||
diff --git a/src/cmd/cgo/main.go b/src/cmd/cgo/main.go
|
||||
index 14642b7576b0..36e76908c602 100644
|
||||
--- a/src/cmd/cgo/main.go
|
||||
+++ b/src/cmd/cgo/main.go
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
+ "sync"
|
||||
|
||||
"cmd/internal/edit"
|
||||
"cmd/internal/objabi"
|
||||
@@ -33,6 +34,8 @@ import (
|
||||
|
||||
// A Package collects information about the package we're going to write.
|
||||
type Package struct {
|
||||
+ workerID int
|
||||
+
|
||||
PackageName string // name of package
|
||||
PackagePath string
|
||||
PtrSize int64
|
||||
@@ -40,17 +43,36 @@ type Package struct {
|
||||
GccOptions []string
|
||||
GccIsClang bool
|
||||
CgoFlags map[string][]string // #cgo flags (CFLAGS, LDFLAGS)
|
||||
- Written map[string]bool
|
||||
- Name map[string]*Name // accumulated Name from Files
|
||||
+ Written *WrittenFiles
|
||||
+ Name map[string]*Name // TODO: parallelize. accumulated Name from Files
|
||||
ExpFunc []*ExpFunc // accumulated ExpFunc from Files
|
||||
- Decl []ast.Decl
|
||||
- GoFiles []string // list of Go files
|
||||
- GccFiles []string // list of gcc output files
|
||||
- Preamble string // collected preamble for _cgo_export.h
|
||||
+ Decl []ast.Decl `json:"-"`
|
||||
+ GoFiles []string // list of Go files
|
||||
+ GccFiles []string // list of gcc output files
|
||||
+ Preamble string // collected preamble for _cgo_export.h
|
||||
+
|
||||
typedefs map[string]bool // type names that appear in the types of the objects we're interested in
|
||||
typedefList []typedefInfo
|
||||
}
|
||||
|
||||
+type WrittenFiles struct {
|
||||
+ mut sync.Mutex
|
||||
+ written map[string]struct{}
|
||||
+}
|
||||
+
|
||||
+func (w *WrittenFiles) Mark(file string) (shouldWrite bool) {
|
||||
+ w.mut.Lock()
|
||||
+ defer w.mut.Unlock()
|
||||
+
|
||||
+ _, already := w.written[file]
|
||||
+ if already {
|
||||
+ return false
|
||||
+ }
|
||||
+
|
||||
+ w.written[file] = struct{}{}
|
||||
+ return true
|
||||
+}
|
||||
+
|
||||
// A typedefInfo is an element on Package.typedefList: a typedef name
|
||||
// and the position where it was required.
|
||||
type typedefInfo struct {
|
||||
@@ -87,7 +109,7 @@ func nameKeys(m map[string]*Name) []string {
|
||||
|
||||
// A Call refers to a call of a C.xxx function in the AST.
|
||||
type Call struct {
|
||||
- Call *ast.CallExpr
|
||||
+ Call *ast.CallExpr `json:"-"`
|
||||
Deferred bool
|
||||
Done bool
|
||||
}
|
||||
@@ -95,7 +117,7 @@ type Call struct {
|
||||
// A Ref refers to an expression of the form C.xxx in the AST.
|
||||
type Ref struct {
|
||||
Name *Name
|
||||
- Expr *ast.Expr
|
||||
+ Expr *ast.Expr `json:"-"`
|
||||
Context astContext
|
||||
Done bool
|
||||
}
|
||||
@@ -108,13 +130,13 @@ var nameKinds = []string{"iconst", "fconst", "sconst", "type", "var", "fpvar", "
|
||||
|
||||
// A Name collects information about C.xxx.
|
||||
type Name struct {
|
||||
- Go string // name used in Go referring to package C
|
||||
- Mangle string // name used in generated Go
|
||||
- C string // name used in C
|
||||
- Define string // #define expansion
|
||||
- Kind string // one of the nameKinds
|
||||
- Type *Type // the type of xxx
|
||||
- FuncType *FuncType
|
||||
+ Go string // name used in Go referring to package C
|
||||
+ Mangle string // name used in generated Go
|
||||
+ C string // name used in C
|
||||
+ Define string // #define expansion
|
||||
+ Kind string // one of the nameKinds
|
||||
+ Type *Type `json:"-"` // the type of xxx
|
||||
+ FuncType *FuncType `json:"-"`
|
||||
AddError bool
|
||||
Const string // constant definition
|
||||
}
|
||||
@@ -133,23 +155,23 @@ func (n *Name) IsConst() bool {
|
||||
// Such functions are identified in the Go input file
|
||||
// by doc comments containing the line //export ExpName
|
||||
type ExpFunc struct {
|
||||
- Func *ast.FuncDecl
|
||||
- ExpName string // name to use from C
|
||||
+ Func *ast.FuncDecl `json:"-"`
|
||||
+ ExpName string // name to use from C
|
||||
Doc string
|
||||
}
|
||||
|
||||
// A TypeRepr contains the string representation of a type.
|
||||
type TypeRepr struct {
|
||||
Repr string
|
||||
- FormatArgs []interface{}
|
||||
+ FormatArgs []interface{} `json:"-"`
|
||||
}
|
||||
|
||||
// A Type collects information about a type in both the C and Go worlds.
|
||||
type Type struct {
|
||||
Size int64
|
||||
Align int64
|
||||
- C *TypeRepr
|
||||
- Go ast.Expr
|
||||
+ C *TypeRepr `json:"-"`
|
||||
+ Go ast.Expr `json:"-"`
|
||||
EnumValues map[string]int64
|
||||
Typedef string
|
||||
BadPointer bool // this pointer type should be represented as a uintptr (deprecated)
|
||||
@@ -223,6 +245,8 @@ var cPrefix string
|
||||
|
||||
var fset = token.NewFileSet()
|
||||
|
||||
+var nparallel = flag.Int("nparallel", runtime.GOMAXPROCS(0), "maximum parallel jobs")
|
||||
+
|
||||
var dynobj = flag.String("dynimport", "", "if non-empty, print dynamic import data for that file")
|
||||
var dynout = flag.String("dynout", "", "write -dynimport output to this file")
|
||||
var dynpackage = flag.String("dynpackage", "main", "set Go package for -dynimport output")
|
||||
@@ -241,11 +265,13 @@ var exportHeader = flag.String("exportheader", "", "where to write export header
|
||||
var gccgo = flag.Bool("gccgo", false, "generate files for use with gccgo")
|
||||
var gccgoprefix = flag.String("gccgoprefix", "", "-fgo-prefix option used with gccgo")
|
||||
var gccgopkgpath = flag.String("gccgopkgpath", "", "-fgo-pkgpath option used with gccgo")
|
||||
-var gccgoMangler func(string) string
|
||||
var importRuntimeCgo = flag.Bool("import_runtime_cgo", true, "import runtime/cgo in generated code")
|
||||
var importSyscall = flag.Bool("import_syscall", true, "import syscall in generated code")
|
||||
var trimpath = flag.String("trimpath", "", "applies supplied rewrites or trims prefixes to recorded source file paths")
|
||||
|
||||
+var gccgoMangler func(string) string
|
||||
+var gccgoManglerOnce sync.Once
|
||||
+
|
||||
var goarch, goos, gomips, gomips64 string
|
||||
var gccBaseCmd []string
|
||||
|
||||
@@ -302,7 +328,7 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
- p := newPackage(args[:i])
|
||||
+ p := newPackage(args[:i], 0)
|
||||
|
||||
// We need a C compiler to be available. Check this.
|
||||
var err error
|
||||
@@ -370,43 +396,152 @@ func main() {
|
||||
}
|
||||
*objDir += string(filepath.Separator)
|
||||
|
||||
- for i, input := range goFiles {
|
||||
- f := fs[i]
|
||||
- p.Translate(f)
|
||||
- for _, cref := range f.Ref {
|
||||
- switch cref.Context {
|
||||
- case ctxCall, ctxCall2:
|
||||
- if cref.Name.Kind != "type" {
|
||||
- break
|
||||
- }
|
||||
- old := *cref.Expr
|
||||
- *cref.Expr = cref.Name.Type.Go
|
||||
- f.Edit.Replace(f.offset(old.Pos()), f.offset(old.End()), gofmt(cref.Name.Type.Go))
|
||||
- }
|
||||
- }
|
||||
- if nerrors > 0 {
|
||||
- os.Exit(2)
|
||||
- }
|
||||
- p.PackagePath = f.Package
|
||||
- p.Record(f)
|
||||
- if *godefs {
|
||||
- os.Stdout.WriteString(p.godefs(f))
|
||||
- } else {
|
||||
- p.writeOutput(f, input)
|
||||
+ pkgs := doFiles(p, goFiles, fs)
|
||||
+
|
||||
+ if !*godefs {
|
||||
+ pkg := coalescePackages(pkgs)
|
||||
+ pkg.writeDefs()
|
||||
+ }
|
||||
+
|
||||
+ if nerrors.get() > 0 {
|
||||
+ os.Exit(2)
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+type doFileJob struct {
|
||||
+ input string
|
||||
+ file *File
|
||||
+}
|
||||
+
|
||||
+func doFiles(p *Package, inputs []string, files []*File) []*Package {
|
||||
+ if len(inputs) != len(files) {
|
||||
+ fatalf("unexpected len(inputs) %d != len(files) %d", len(inputs), len(files))
|
||||
+ }
|
||||
+
|
||||
+ jobCh := make(chan doFileJob)
|
||||
+
|
||||
+ var wg sync.WaitGroup
|
||||
+ wg.Add(*nparallel)
|
||||
+
|
||||
+ pkgs := make([]*Package, *nparallel)
|
||||
+ pkgs[0] = p
|
||||
+
|
||||
+ for n := 1; n < *nparallel; n++ {
|
||||
+ pkgs[n] = cloneNewPackage(p, n)
|
||||
+ }
|
||||
+
|
||||
+ for n := 0; n < *nparallel; n++ {
|
||||
+ p := pkgs[n]
|
||||
+ go func() {
|
||||
+ doFileWorker(p, jobCh)
|
||||
+ wg.Done()
|
||||
+ }()
|
||||
+ }
|
||||
+
|
||||
+ for i := range inputs {
|
||||
+ jobCh <- doFileJob{
|
||||
+ input: inputs[i],
|
||||
+ file: files[i],
|
||||
}
|
||||
}
|
||||
|
||||
- if !*godefs {
|
||||
- p.writeDefs()
|
||||
+ close(jobCh)
|
||||
+ wg.Wait()
|
||||
+
|
||||
+ return pkgs
|
||||
+}
|
||||
+
|
||||
+func doFileWorker(p *Package, jobCh <-chan doFileJob) *Package {
|
||||
+ for job := range jobCh {
|
||||
+ doFile(p, job.input, job.file)
|
||||
}
|
||||
- if nerrors > 0 {
|
||||
+
|
||||
+ return p
|
||||
+}
|
||||
+
|
||||
+func doFile(p *Package, input string, f *File) {
|
||||
+ p.Translate(f)
|
||||
+ for _, cref := range f.Ref {
|
||||
+ switch cref.Context {
|
||||
+ case ctxCall, ctxCall2:
|
||||
+ if cref.Name.Kind != "type" {
|
||||
+ break
|
||||
+ }
|
||||
+ old := *cref.Expr
|
||||
+ *cref.Expr = cref.Name.Type.Go
|
||||
+ f.Edit.Replace(f.offset(old.Pos()), f.offset(old.End()), gofmt(cref.Name.Type.Go))
|
||||
+ }
|
||||
+ }
|
||||
+ if nerrors.get() > 0 {
|
||||
os.Exit(2)
|
||||
}
|
||||
+ p.PackagePath = f.Package
|
||||
+ p.Record(f)
|
||||
+ if *godefs {
|
||||
+ os.Stdout.WriteString(p.godefs(f))
|
||||
+ } else {
|
||||
+ p.writeOutput(f, input)
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func cloneNewPackage(p *Package, workerID int) *Package {
|
||||
+ return &Package{
|
||||
+ workerID: workerID,
|
||||
+ PackageName: p.PackageName,
|
||||
+ PackagePath: p.PackagePath,
|
||||
+ PtrSize: p.PtrSize,
|
||||
+ IntSize: p.IntSize,
|
||||
+ GccOptions: p.GccOptions, // init in main only
|
||||
+ GccIsClang: p.GccIsClang,
|
||||
+ CgoFlags: p.CgoFlags, // init in main only
|
||||
+ Written: p.Written,
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+func coalescePackages(pkgs []*Package) *Package {
|
||||
+ if len(pkgs) == 0 {
|
||||
+ return nil
|
||||
+ }
|
||||
+
|
||||
+ nonEmpty := pkgs[:0]
|
||||
+ for _, p := range pkgs {
|
||||
+ if p.PackageName != "" {
|
||||
+ nonEmpty = append(nonEmpty, p)
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ pkgs = nonEmpty
|
||||
+ pkg := pkgs[0]
|
||||
+
|
||||
+ for _, p := range pkgs[1:] {
|
||||
+ pkg.PackageName = p.PackageName
|
||||
+ pkg.PackagePath = p.PackagePath
|
||||
+ pkg.ExpFunc = append(pkg.ExpFunc, p.ExpFunc...)
|
||||
+ pkg.Decl = append(pkg.Decl, p.Decl...)
|
||||
+ pkg.GoFiles = append(pkg.GoFiles, p.GoFiles...)
|
||||
+ pkg.GccFiles = append(pkg.GccFiles, p.GccFiles...)
|
||||
+ pkg.Preamble += p.Preamble
|
||||
+
|
||||
+ if pkg.Name == nil && p.Name != nil {
|
||||
+ pkg.Name = p.Name
|
||||
+ continue
|
||||
+ }
|
||||
+
|
||||
+ for k, v := range p.Name {
|
||||
+ if o, ok := pkg.Name[k]; ok && !reflect.DeepEqual(v, o) {
|
||||
+ fatalf("duplicate pkg.Name %s, %v == %v", k, v, o)
|
||||
+ }
|
||||
+
|
||||
+ pkg.Name[k] = v
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return pkg
|
||||
}
|
||||
|
||||
// newPackage returns a new Package that will invoke
|
||||
// gcc with the additional arguments specified in args.
|
||||
-func newPackage(args []string) *Package {
|
||||
+func newPackage(args []string, workerID int) *Package {
|
||||
goarch = runtime.GOARCH
|
||||
if s := os.Getenv("GOARCH"); s != "" {
|
||||
goarch = s
|
||||
@@ -432,10 +567,13 @@ func newPackage(args []string) *Package {
|
||||
os.Setenv("LC_ALL", "C")
|
||||
|
||||
p := &Package{
|
||||
+ workerID: workerID,
|
||||
PtrSize: ptrSize,
|
||||
IntSize: intSize,
|
||||
CgoFlags: make(map[string][]string),
|
||||
- Written: make(map[string]bool),
|
||||
+ Written: &WrittenFiles{
|
||||
+ written: make(map[string]struct{}),
|
||||
+ },
|
||||
}
|
||||
p.addToFlag("CFLAGS", args)
|
||||
return p
|
||||
diff --git a/src/cmd/cgo/out.go b/src/cmd/cgo/out.go
|
||||
index 4968f7059d9b..b934d12d9423 100644
|
||||
--- a/src/cmd/cgo/out.go
|
||||
+++ b/src/cmd/cgo/out.go
|
||||
@@ -32,14 +32,13 @@ var (
|
||||
|
||||
// writeDefs creates output files to be compiled by gc and gcc.
|
||||
func (p *Package) writeDefs() {
|
||||
- var fgo2, fc io.Writer
|
||||
- f := creat(*objDir + "_cgo_gotypes.go")
|
||||
- defer f.Close()
|
||||
- fgo2 = f
|
||||
+ fgo2 := creat(*objDir + "_cgo_gotypes.go")
|
||||
+ defer fgo2.Close()
|
||||
+
|
||||
+ var fc *os.File
|
||||
if *gccgo {
|
||||
- f := creat(*objDir + "_cgo_defun.c")
|
||||
- defer f.Close()
|
||||
- fc = f
|
||||
+ fc = creat(*objDir + "_cgo_defun.c")
|
||||
+ defer fc.Close()
|
||||
}
|
||||
fm := creat(*objDir + "_cgo_main.c")
|
||||
|
||||
@@ -682,12 +681,11 @@ var isBuiltin = map[string]bool{
|
||||
|
||||
func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) {
|
||||
name := n.Mangle
|
||||
- if isBuiltin[name] || p.Written[name] {
|
||||
+ if isBuiltin[name] || !p.Written.Mark(name) {
|
||||
// The builtins are already defined in the C prolog, and we don't
|
||||
// want to duplicate function definitions we've already done.
|
||||
return
|
||||
}
|
||||
- p.Written[name] = true
|
||||
|
||||
if *gccgo {
|
||||
p.writeGccgoOutputFunc(fgcc, n)
|
||||
@@ -1294,7 +1292,7 @@ func (p *Package) writeExportHeader(fgcch io.Writer) {
|
||||
|
||||
// gccgoToSymbol converts a name to a mangled symbol for gccgo.
|
||||
func gccgoToSymbol(ppath string) string {
|
||||
- if gccgoMangler == nil {
|
||||
+ gccgoManglerOnce.Do(func() {
|
||||
var err error
|
||||
cmd := os.Getenv("GCCGO")
|
||||
if cmd == "" {
|
||||
@@ -1307,7 +1305,7 @@ func gccgoToSymbol(ppath string) string {
|
||||
if err != nil {
|
||||
fatalf("%v", err)
|
||||
}
|
||||
- }
|
||||
+ })
|
||||
return gccgoMangler(ppath)
|
||||
}
|
||||
|
||||
diff --git a/src/cmd/cgo/util.go b/src/cmd/cgo/util.go
|
||||
index 00d931b98a0c..bea77b841466 100644
|
||||
--- a/src/cmd/cgo/util.go
|
||||
+++ b/src/cmd/cgo/util.go
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
exec "internal/execabs"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
+ "sync/atomic"
|
||||
)
|
||||
|
||||
// run runs the command argv, feeding in stdin on standard input.
|
||||
@@ -87,16 +88,22 @@ func lineno(pos token.Pos) string {
|
||||
func fatalf(msg string, args ...interface{}) {
|
||||
// If we've already printed other errors, they might have
|
||||
// caused the fatal condition. Assume they're enough.
|
||||
- if nerrors == 0 {
|
||||
+ if !nerrors.has() {
|
||||
fmt.Fprintf(os.Stderr, "cgo: "+msg+"\n", args...)
|
||||
}
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
-var nerrors int
|
||||
+type atomicNErrors uint64
|
||||
+
|
||||
+var nerrors atomicNErrors
|
||||
+
|
||||
+func (n *atomicNErrors) add() { atomic.AddUint64((*uint64)(n), 1) }
|
||||
+func (n *atomicNErrors) has() bool { return n.get() > 0 }
|
||||
+func (n *atomicNErrors) get() uint64 { return atomic.LoadUint64((*uint64)(n)) }
|
||||
|
||||
func error_(pos token.Pos, msg string, args ...interface{}) {
|
||||
- nerrors++
|
||||
+ nerrors.add()
|
||||
if pos.IsValid() {
|
||||
fmt.Fprintf(os.Stderr, "%s: ", fset.Position(pos).String())
|
||||
} else {
|
||||
diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go
|
||||
index 0af39a991c02..ac658f4803ef 100644
|
||||
--- a/src/cmd/go/internal/work/exec.go
|
||||
+++ b/src/cmd/go/internal/work/exec.go
|
||||
@@ -2821,7 +2821,12 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo
|
||||
cgoflags = append(cgoflags, "-trimpath", strings.Join(trimpath, ";"))
|
||||
}
|
||||
|
||||
- if err := b.run(a, execdir, p.ImportPath, cgoenv, cfg.BuildToolexec, cgoExe, "-objdir", objdir, "-importpath", p.ImportPath, cgoflags, "--", cgoCPPFLAGS, cgoCFLAGS, cgofiles); err != nil {
|
||||
+ nparallel := runtime.GOMAXPROCS(0)
|
||||
+ if nparallel > 4 {
|
||||
+ nparallel = 4
|
||||
+ }
|
||||
+
|
||||
+ if err := b.run(a, execdir, p.ImportPath, cgoenv, cfg.BuildToolexec, cgoExe, "-nparallel", strconv.Itoa(nparallel), "-objdir", objdir, "-importpath", p.ImportPath, cgoflags, "--", cgoCPPFLAGS, cgoCFLAGS, cgofiles); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
outGo = append(outGo, gofiles...)
|
@ -0,0 +1,159 @@
|
||||
From ec3e1c9471f170187b6a7c83ab0364253f895c28 Mon Sep 17 00:00:00 2001
|
||||
From: diamondburned <datutbrus@gmail.com>
|
||||
Date: Thu, 5 Aug 2021 16:43:01 -0700
|
||||
Subject: [PATCH] cmd/go/internal/work: concurrent ccompile routines
|
||||
|
||||
This commit allows execution of gcc and others after the cgo stage to be
|
||||
concurrent.
|
||||
|
||||
Prior to this change, only 1 gcc instance will run at a time.
|
||||
|
||||
After this, these instances will be launched off in multiples at the
|
||||
same time, where the parallelism is determined by the thread count, but
|
||||
capped at 4 using gcBackendConcurrency to calculate.
|
||||
---
|
||||
src/cmd/go/internal/work/exec.go | 97 +++++++++++++++++++++++++++-----
|
||||
1 file changed, 82 insertions(+), 15 deletions(-)
|
||||
|
||||
diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go
|
||||
index ac80f503cd89..0af39a991c02 100644
|
||||
--- a/src/cmd/go/internal/work/exec.go
|
||||
+++ b/src/cmd/go/internal/work/exec.go
|
||||
@@ -2838,48 +2838,70 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo
|
||||
return objdir + fmt.Sprintf("_x%03d.o", oseq)
|
||||
}
|
||||
|
||||
+ jobCh := make(chan func() error)
|
||||
+ jobErrCh := make(chan error)
|
||||
+
|
||||
+ nparallel = gccConcurrency(a)
|
||||
+ go func() { jobErrCh <- runJobDispatcher(nparallel, jobCh) }()
|
||||
+
|
||||
// gcc
|
||||
cflags := str.StringList(cgoCPPFLAGS, cgoCFLAGS)
|
||||
for _, cfile := range cfiles {
|
||||
ofile := nextOfile()
|
||||
- if err := b.gcc(a, p, a.Objdir, ofile, cflags, objdir+cfile); err != nil {
|
||||
- return nil, nil, err
|
||||
- }
|
||||
+ cfile := cfile
|
||||
outObj = append(outObj, ofile)
|
||||
+
|
||||
+ jobCh <- func() error {
|
||||
+ return b.gcc(a, p, a.Objdir, ofile, cflags, objdir+cfile)
|
||||
+ }
|
||||
}
|
||||
|
||||
for _, file := range gccfiles {
|
||||
ofile := nextOfile()
|
||||
- if err := b.gcc(a, p, a.Objdir, ofile, cflags, file); err != nil {
|
||||
- return nil, nil, err
|
||||
- }
|
||||
+ ifile := file
|
||||
outObj = append(outObj, ofile)
|
||||
+
|
||||
+ jobCh <- func() error {
|
||||
+ return b.gcc(a, p, a.Objdir, ofile, cflags, ifile)
|
||||
+ }
|
||||
}
|
||||
|
||||
cxxflags := str.StringList(cgoCPPFLAGS, cgoCXXFLAGS)
|
||||
for _, file := range gxxfiles {
|
||||
ofile := nextOfile()
|
||||
- if err := b.gxx(a, p, a.Objdir, ofile, cxxflags, file); err != nil {
|
||||
- return nil, nil, err
|
||||
- }
|
||||
+ ifile := file
|
||||
outObj = append(outObj, ofile)
|
||||
+
|
||||
+ jobCh <- func() error {
|
||||
+ return b.gxx(a, p, a.Objdir, ofile, cxxflags, ifile)
|
||||
+ }
|
||||
}
|
||||
|
||||
for _, file := range mfiles {
|
||||
ofile := nextOfile()
|
||||
- if err := b.gcc(a, p, a.Objdir, ofile, cflags, file); err != nil {
|
||||
- return nil, nil, err
|
||||
- }
|
||||
+ ifile := file
|
||||
outObj = append(outObj, ofile)
|
||||
+
|
||||
+ jobCh <- func() error {
|
||||
+ return b.gcc(a, p, a.Objdir, ofile, cflags, ifile)
|
||||
+ }
|
||||
}
|
||||
|
||||
fflags := str.StringList(cgoCPPFLAGS, cgoFFLAGS)
|
||||
for _, file := range ffiles {
|
||||
ofile := nextOfile()
|
||||
- if err := b.gfortran(a, p, a.Objdir, ofile, fflags, file); err != nil {
|
||||
- return nil, nil, err
|
||||
- }
|
||||
+ ifile := file
|
||||
outObj = append(outObj, ofile)
|
||||
+
|
||||
+ jobCh <- func() error {
|
||||
+ return b.gfortran(a, p, a.Objdir, ofile, fflags, ifile)
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ close(jobCh)
|
||||
+
|
||||
+ if err := <-jobErrCh; err != nil {
|
||||
+ return nil, nil, err
|
||||
}
|
||||
|
||||
switch cfg.BuildToolchainName {
|
||||
@@ -2980,6 +3002,51 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo
|
||||
return outGo, outObj, nil
|
||||
}
|
||||
|
||||
+// gccConcurrency returns the concurrency level for spawning gcc processes.
|
||||
+func gccConcurrency(a *Action) int {
|
||||
+ gcflags := str.StringList(forcedGcflags, a.Package.Internal.Gcflags)
|
||||
+ return gcBackendConcurrency(gcflags)
|
||||
+}
|
||||
+
|
||||
+func runJobDispatcher(nparallel int, jobCh <-chan func() error) error {
|
||||
+ var wg sync.WaitGroup
|
||||
+
|
||||
+ var firstErr error
|
||||
+ var errNum int
|
||||
+ var errMut sync.Mutex
|
||||
+
|
||||
+ semaphore := make(chan struct{}, nparallel)
|
||||
+
|
||||
+ for job := range jobCh {
|
||||
+ job := job
|
||||
+
|
||||
+ wg.Add(1)
|
||||
+ semaphore <- struct{}{}
|
||||
+
|
||||
+ go func() {
|
||||
+ defer wg.Done()
|
||||
+ defer func() { <-semaphore }()
|
||||
+
|
||||
+ if err := job(); err != nil {
|
||||
+ errMut.Lock()
|
||||
+ if firstErr == nil {
|
||||
+ firstErr = err
|
||||
+ }
|
||||
+ errNum++
|
||||
+ errMut.Unlock()
|
||||
+ }
|
||||
+ }()
|
||||
+ }
|
||||
+
|
||||
+ wg.Wait()
|
||||
+
|
||||
+ if errNum > 0 {
|
||||
+ return fmt.Errorf("encountered %d errors, including: %w", errNum, firstErr)
|
||||
+ }
|
||||
+
|
||||
+ return nil
|
||||
+}
|
||||
+
|
||||
// dynimport creates a Go source file named importGo containing
|
||||
// //go:cgo_import_dynamic directives for each symbol or library
|
||||
// dynamically imported by the object files outObj.
|
Loading…
Reference in New Issue
Block a user