flake,go: add,use overlay.nix w/ patched Go

flake.nix

@@ -31,13 +31,18 @@
       forAllSystems = nixpkgs.lib.genAttrs supportedSystems;
 
       # Nixpkgs instantiated for supported system types.
-      nixpkgsFor = forAllSystems (system: import nixpkgs { inherit system; overlays = [ ]; });
+      nixpkgsFor = forAllSystems (system: import nixpkgs {
+        inherit system;
+        overlays = [
+          (import ./overlay.nix)
+        ];
+      });
     in
     rec {
       packages = forAllSystems (system:
         let
           pkgs = nixpkgsFor.${system};
-          inherit (pkgs) lib;# -> lib = pkgs.lib;
+          inherit (pkgs) lib overlays;# -> lib = pkgs.lib;overlays = pkgs.overlays;
         in
         rec {
           go-xkcdreader = with pkgs; buildGoModule rec {
@@ -73,8 +78,8 @@
             ];
 
             modSha256 = lib.fakeSha256;
-            # vendorSha256 = "";
-            vendorSha256 = "sha256-i6VOX1O/IqIOWUzgc/2U/a95KhzAG0/NOIG/D8E0BEk=";
+            # dont't forget to update vendorSha256 whenever go.mod or go.sum change
+            vendorSha256 = "sha256-oHOMkvQhMFsAGgMcAHvxZp1vcDSVLmUYhft+cvnMd6M=";
 
             # In 'nix develop', we don't need a copy of the source tree
             # in the Nix store.
@@ -107,6 +112,13 @@
             inherit system;
             overlays = [ nixgl.overlay ];
           };
+          goPkgs = import nixpkgs {
+            useFetched = true;
+            inherit system;
+            overlays = self.overlays or [ ] ++ [
+              (import ./overlay.nix)
+            ];
+          };
         in
         {
           default = with pkgs; mkShell
@@ -132,7 +144,14 @@
                 # pkgs.nixgl.auto.nixGLDefault # requires --impure
               ];
               packages = [
-                go
+                ## use patched Go, since it's supposed to be faster
+                goPkgs.go
+                ## if you wish to use this, uncomment the related block in
+                ## overlay.nix and the next line
+                # goPkgs.dominikh.go-tools
+                goPkgs.patchelf-x86_64
+                goPkgs.patchelf-aarch64
+
                 gopls
                 gofumpt
                 go-tools

overlay.nix

@@ -0,0 +1,63 @@
+# heavily inspired by https://github.com/diamondburned/gotk4-nix/blob/d2bd6577f1867cb740b281baa48a895aed494967/overlay.nix
+self: super:
+let patchelfer = arch: interpreter: super.writeShellScriptBin
+  "patchelf-${arch}"
+  "${super.patchelf}/bin/patchelf --set-interpreter ${interpreter} \"$@\"";
+in
+{
+  go = super.go.overrideAttrs (old: {
+    version = "1.18.2";
+    src = builtins.fetchurl {
+      url = "https://go.dev/dl/go1.18.2.src.tar.gz";
+      sha256 = "sha256:1qk7as7a5wx3522ldsq8il1il9ic5xhbl6drg89r4h63l8zd0i1c";
+    };
+    doCheck = false;
+    patches = (old.patches or [ ]) ++ [
+      # cmd/go/internal/work: concurrent ccompile routines
+      (super.fetchpatch {
+        url = "https://github.com/diamondburned/go/commit/ec3e1c9471f170187b6a7c83ab0364253f895c28.patch";
+        sha256 = "sha256-KXP3gHRVHCVY60HnrLHv+QLib1hSYZOxsMjkbAiNi1g=";
+      })
+      # cmd/cgo: concurrent file generation
+      (super.fetchpatch {
+        url = "https://github.com/diamondburned/go/commit/50e04befeca9ae63296a73c8d5d2870b904971b4.patch";
+        sha256 = "sha256-6PNUBlhGj53XS7f3dYOisDQ1PJldqwmWcgBnPE0cZsE=";
+      })
+      # the patches are also present locally for archival purposes
+      # ./patches/diamondburned/go/ec3e1c9471f170187b6a7c83ab0364253f895c28.patch
+      # ./patches/diamondburned/go/50e04befeca9ae63296a73c8d5d2870b904971b4.patch
+      # TODO(me): don't forget to download the patch files when bumping the
+      # urls above
+    ];
+  });
+
+  # might as well be 'buildGoModule = super.buildGo118Module.override' but
+  # we're overriding just the go package itself and leaving the rest of the
+  # function alone. letting 'builGoModule' to be updated only with flake's
+  # nixpkgs updates therefore seems to be the more reasonable option.
+  buildGoModule = super.buildGoModule.override {
+    # use the go we just built from sources as 'go' in the buildGoModule func
+    inherit (self) go;
+  };
+
+  # inherit (super) go-tools;
+
+  # See https://sourceware.org/glibc/wiki/ABIList.
+  patchelf-x86_64 = patchelfer "x86_64" "/lib64/ld-linux-x86-64.so.2";
+  patchelf-aarch64 = patchelfer "aarch64" "/lib/ld-linux-aarch64.so.1";
+
+  #dominikh = {
+  #  go-tools = self.buildGoModule {
+  #    name = "dominikh-go-tools";
+  #    src = super.fetchFromGitHub {
+  #      owner = "dominikh";
+  #      repo = "go-tools";
+  #      rev = "f4a2f64ce2386d1d392f2da44043c5ee3fb48216";
+  #      sha256 = "1yhbz2sf332b6i00slsj4cn8r66x27kddw5vcjygkkiyny1a99qb";
+  #    };
+  #    vendorSha256 = "09jbarlbq47pcxy5zkja8gqvnqjp2mpbxnciv9lhilw9swqqwc0j";
+  #    doCheck = false;
+  #    subPackages = [ "cmd/staticcheck" ];
+  #  };
+  #};
+}

patches/diamondburned/go/50e04befeca9ae63296a73c8d5d2870b904971b4.patch

@@ -0,0 +1,883 @@
+From 50e04befeca9ae63296a73c8d5d2870b904971b4 Mon Sep 17 00:00:00 2001
+From: diamondburned <datutbrus@gmail.com>
+Date: Thu, 5 Aug 2021 16:47:49 -0700
+Subject: [PATCH] cmd/cgo: concurrent file generation
+
+This commit allows cmd/cgo to generate files in parallel to each other.
+The parallelism is determined by either the thread count (using
+$GOMAXPROCS) or the -nparallel flag, which is set in
+cmd/go/internal/work to be maximum 4.
+
+The commit addresses some existing race conditions that had to do with
+mutating the global state by using a global write mutex during the
+translation stage.
+
+Other race conditions that had to do with mutating the Package singleton
+was solved by giving each translation worker its own package, and then
+combining them back into a single package.
+
+With this and the cmd/go/internal/work changes in place, build time
+(from scratch, with -a) for gotktrix is reduced from 33 minutes to 10
+minutes on an 8-thread Intel Core i5-8250U.
+---
+ src/cmd/cgo/gcc.go               | 120 +++++++++++-----
+ src/cmd/cgo/godefs.go            |   6 +-
+ src/cmd/cgo/main.go              | 238 ++++++++++++++++++++++++-------
+ src/cmd/cgo/out.go               |  20 ++-
+ src/cmd/cgo/util.go              |  13 +-
+ src/cmd/go/internal/work/exec.go |   7 +-
+ 6 files changed, 303 insertions(+), 101 deletions(-)
+
+diff --git a/src/cmd/cgo/gcc.go b/src/cmd/cgo/gcc.go
+index 997a830994f0..59a8405c0c71 100644
+--- a/src/cmd/cgo/gcc.go
++++ b/src/cmd/cgo/gcc.go
+@@ -26,6 +26,8 @@ import (
+ 	"os/exec"
+ 	"strconv"
+ 	"strings"
++	"sync"
++	"sync/atomic"
+ 	"unicode"
+ 	"unicode/utf8"
+ 
+@@ -485,7 +487,7 @@ func (p *Package) guessKinds(f *File) []*Name {
+ 		}
+ 		needType = append(needType, n)
+ 	}
+-	if nerrors > 0 {
++	if nerrors.has() {
+ 		// Check if compiling the preamble by itself causes any errors,
+ 		// because the messages we've printed out so far aren't helpful
+ 		// to users debugging preamble mistakes. See issue 8442.
+@@ -727,9 +729,12 @@ func (p *Package) prepareNames(f *File) {
+ 			}
+ 		}
+ 		p.mangleName(n)
++
++		globalMu.Lock()
+ 		if n.Kind == "type" && typedef[n.Mangle] == nil {
+ 			typedef[n.Mangle] = n.Type
+ 		}
++		globalMu.Unlock()
+ 	}
+ }
+ 
+@@ -1023,7 +1028,7 @@ func (p *Package) hasPointer(f *File, t ast.Expr, top bool) bool {
+ 		}
+ 		// Check whether this is a pointer to a C union (or class)
+ 		// type that contains a pointer.
+-		if unionWithPointer[t.X] {
++		if isUnionWithPointer(t.X) {
+ 			return true
+ 		}
+ 		return p.hasPointer(f, t.X, false)
+@@ -1046,7 +1051,10 @@ func (p *Package) hasPointer(f *File, t ast.Expr, top bool) bool {
+ 				}
+ 			}
+ 		}
+-		if def := typedef[t.Name]; def != nil {
++		globalMu.Lock()
++		def := typedef[t.Name]
++		globalMu.Unlock()
++		if def != nil {
+ 			return p.hasPointer(f, def.Go, top)
+ 		}
+ 		if t.Name == "string" {
+@@ -1582,7 +1590,7 @@ func checkGCCBaseCmd() ([]string, error) {
+ }
+ 
+ // gccMachine returns the gcc -m flag to use, either "-m32", "-m64" or "-marm".
+-func (p *Package) gccMachine() []string {
++func gccMachine() []string {
+ 	switch goarch {
+ 	case "amd64":
+ 		if goos == "darwin" {
+@@ -1617,20 +1625,20 @@ func (p *Package) gccMachine() []string {
+ 	return nil
+ }
+ 
+-func gccTmp() string {
+-	return *objDir + "_cgo_.o"
++func (p *Package) gccTmp() string {
++	return fmt.Sprintf("%s_%d_cgo_.o", *objDir, p.workerID)
+ }
+ 
+ // gccCmd returns the gcc command line to use for compiling
+ // the input.
+ func (p *Package) gccCmd() []string {
+ 	c := append(gccBaseCmd,
+-		"-w",          // no warnings
+-		"-Wno-error",  // warnings are not errors
+-		"-o"+gccTmp(), // write object to tmp
+-		"-gdwarf-2",   // generate DWARF v2 debugging symbols
+-		"-c",          // do not link
+-		"-xc",         // input language is C
++		"-w",            // no warnings
++		"-Wno-error",    // warnings are not errors
++		"-o"+p.gccTmp(), // write object to tmp
++		"-gdwarf-2",     // generate DWARF v2 debugging symbols
++		"-c",            // do not link
++		"-xc",           // input language is C
+ 	)
+ 	if p.GccIsClang {
+ 		c = append(c,
+@@ -1653,7 +1661,7 @@ func (p *Package) gccCmd() []string {
+ 	}
+ 
+ 	c = append(c, p.GccOptions...)
+-	c = append(c, p.gccMachine()...)
++	c = append(c, gccMachine()...)
+ 	if goos == "aix" {
+ 		c = append(c, "-maix64")
+ 		c = append(c, "-mcmodel=large")
+@@ -1717,11 +1725,11 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
+ 		}
+ 	}
+ 
+-	if f, err := macho.Open(gccTmp()); err == nil {
++	if f, err := macho.Open(p.gccTmp()); err == nil {
+ 		defer f.Close()
+ 		d, err := f.DWARF()
+ 		if err != nil {
+-			fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
++			fatalf("cannot load DWARF output from %s: %v", p.gccTmp(), err)
+ 		}
+ 		bo := f.ByteOrder
+ 		if f.Symtab != nil {
+@@ -1795,11 +1803,11 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
+ 		return d, ints, floats, strs
+ 	}
+ 
+-	if f, err := elf.Open(gccTmp()); err == nil {
++	if f, err := elf.Open(p.gccTmp()); err == nil {
+ 		defer f.Close()
+ 		d, err := f.DWARF()
+ 		if err != nil {
+-			fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
++			fatalf("cannot load DWARF output from %s: %v", p.gccTmp(), err)
+ 		}
+ 		bo := f.ByteOrder
+ 		symtab, err := f.Symbols()
+@@ -1874,11 +1882,11 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
+ 		return d, ints, floats, strs
+ 	}
+ 
+-	if f, err := pe.Open(gccTmp()); err == nil {
++	if f, err := pe.Open(p.gccTmp()); err == nil {
+ 		defer f.Close()
+ 		d, err := f.DWARF()
+ 		if err != nil {
+-			fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
++			fatalf("cannot load DWARF output from %s: %v", p.gccTmp(), err)
+ 		}
+ 		bo := binary.LittleEndian
+ 		for _, s := range f.Symbols {
+@@ -1946,11 +1954,11 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
+ 		return d, ints, floats, strs
+ 	}
+ 
+-	if f, err := xcoff.Open(gccTmp()); err == nil {
++	if f, err := xcoff.Open(p.gccTmp()); err == nil {
+ 		defer f.Close()
+ 		d, err := f.DWARF()
+ 		if err != nil {
+-			fatalf("cannot load DWARF output from %s: %v", gccTmp(), err)
++			fatalf("cannot load DWARF output from %s: %v", p.gccTmp(), err)
+ 		}
+ 		bo := binary.BigEndian
+ 		for _, s := range f.Symbols {
+@@ -2016,7 +2024,7 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
+ 		buildStrings()
+ 		return d, ints, floats, strs
+ 	}
+-	fatalf("cannot parse gcc output %s as ELF, Mach-O, PE, XCOFF object", gccTmp())
++	fatalf("cannot parse gcc output %s as ELF, Mach-O, PE, XCOFF object", p.gccTmp())
+ 	panic("not reached")
+ }
+ 
+@@ -2026,7 +2034,7 @@ func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int6
+ // and its included files.
+ func (p *Package) gccDefines(stdin []byte) string {
+ 	base := append(gccBaseCmd, "-E", "-dM", "-xc")
+-	base = append(base, p.gccMachine()...)
++	base = append(base, gccMachine()...)
+ 	stdout, _ := runGcc(stdin, append(append(base, p.GccOptions...), "-"))
+ 	return stdout
+ }
+@@ -2126,7 +2134,13 @@ type typeConv struct {
+ 	intSize int64
+ }
+ 
+-var tagGen int
++var tagGenCounter int64
++
++func genTag() int {
++	return int(atomic.AddInt64(&tagGenCounter, 1) - 1)
++}
++
++var globalMu sync.Mutex
+ var typedef = make(map[string]*Type)
+ var goIdent = make(map[string]*ast.Ident)
+ 
+@@ -2134,6 +2148,29 @@ var goIdent = make(map[string]*ast.Ident)
+ // that may contain a pointer. This is used for cgo pointer checking.
+ var unionWithPointer = make(map[ast.Expr]bool)
+ 
++func unionWithPointerTrueIfKey(setKey, ifKey ast.Expr) {
++	globalMu.Lock()
++	defer globalMu.Unlock()
++
++	if unionWithPointer[ifKey] {
++		unionWithPointer[setKey] = true
++	}
++}
++
++func isUnionWithPointer(key ast.Expr) bool {
++	globalMu.Lock()
++	defer globalMu.Unlock()
++
++	return unionWithPointer[key]
++}
++
++func unionWithPointerTrue(key ast.Expr) {
++	globalMu.Lock()
++	defer globalMu.Unlock()
++
++	unionWithPointer[key] = true
++}
++
+ // anonymousStructTag provides a consistent tag for an anonymous struct.
+ // The same dwarf.StructType pointer will always get the same tag.
+ var anonymousStructTag = make(map[*dwarf.StructType]string)
+@@ -2472,9 +2509,7 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
+ 		t.Size = t1.Size
+ 		t.Align = t1.Align
+ 		t.Go = t1.Go
+-		if unionWithPointer[t1.Go] {
+-			unionWithPointer[t.Go] = true
+-		}
++		unionWithPointerTrueIfKey(t.Go, t1.Go)
+ 		t.EnumValues = nil
+ 		t.Typedef = ""
+ 		t.C.Set("%s "+dt.Qual, t1.C)
+@@ -2488,18 +2523,21 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
+ 			break
+ 		}
+ 		if tag == "" {
++			globalMu.Lock()
+ 			tag = anonymousStructTag[dt]
+ 			if tag == "" {
+-				tag = "__" + strconv.Itoa(tagGen)
+-				tagGen++
++				tag = "__" + strconv.Itoa(genTag())
+ 				anonymousStructTag[dt] = tag
+ 			}
++			globalMu.Unlock()
+ 		} else if t.C.Empty() {
+ 			t.C.Set(dt.Kind + " " + tag)
+ 		}
+ 		name := c.Ident("_Ctype_" + dt.Kind + "_" + tag)
+ 		t.Go = name // publish before recursive calls
++		globalMu.Lock()
+ 		goIdent[name.Name] = name
++		globalMu.Unlock()
+ 		if dt.ByteSize < 0 {
+ 			// Size calculation in c.Struct/c.Opaque will die with size=-1 (unknown),
+ 			// so execute the basic things that the struct case would do
+@@ -2519,20 +2557,24 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
+ 				// on the Go heap, right? It currently doesn't work for unions because
+ 				// they are defined as a type alias for struct{}, not a defined type.
+ 			}
++			globalMu.Lock()
+ 			typedef[name.Name] = &tt
++			globalMu.Unlock()
+ 			break
+ 		}
+ 		switch dt.Kind {
+ 		case "class", "union":
+ 			t.Go = c.Opaque(t.Size)
+ 			if c.dwarfHasPointer(dt, pos) {
+-				unionWithPointer[t.Go] = true
++				unionWithPointerTrue(t.Go)
+ 			}
+ 			if t.C.Empty() {
+ 				t.C.Set("__typeof__(unsigned char[%d])", t.Size)
+ 			}
+ 			t.Align = 1 // TODO: should probably base this on field alignment.
++			globalMu.Lock()
+ 			typedef[name.Name] = t
++			globalMu.Unlock()
+ 		case "struct":
+ 			g, csyntax, align := c.Struct(dt, pos)
+ 			if t.C.Empty() {
+@@ -2545,7 +2587,9 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
+ 			}
+ 			tt.Go = g
+ 			tt.NotInHeap = c.notInHeapStructs[tag]
++			globalMu.Lock()
+ 			typedef[name.Name] = &tt
++			globalMu.Unlock()
+ 		}
+ 
+ 	case *dwarf.TypedefType:
+@@ -2568,7 +2612,9 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
+ 			break
+ 		}
+ 		name := c.Ident("_Ctype_" + dt.Name)
++		globalMu.Lock()
+ 		goIdent[name.Name] = name
++		globalMu.Unlock()
+ 		akey := ""
+ 		if c.anonymousStructTypedef(dt) {
+ 			// only load type recursively for typedefs of anonymous
+@@ -2583,10 +2629,12 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
+ 			s.BadPointer = true
+ 			sub = &s
+ 			// Make sure we update any previously computed type.
++			globalMu.Lock()
+ 			if oldType := typedef[name.Name]; oldType != nil {
+ 				oldType.Go = sub.Go
+ 				oldType.BadPointer = true
+ 			}
++			globalMu.Unlock()
+ 		}
+ 		if c.badVoidPointerTypedef(dt) {
+ 			// Treat this typedef as a pointer to a NotInHeap void.
+@@ -2615,11 +2663,10 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
+ 		t.Go = name
+ 		t.BadPointer = sub.BadPointer
+ 		t.NotInHeap = sub.NotInHeap
+-		if unionWithPointer[sub.Go] {
+-			unionWithPointer[t.Go] = true
+-		}
++		unionWithPointerTrueIfKey(t.Go, sub.Go)
+ 		t.Size = sub.Size
+ 		t.Align = sub.Align
++		globalMu.Lock()
+ 		oldType := typedef[name.Name]
+ 		if oldType == nil {
+ 			tt := *t
+@@ -2628,6 +2675,7 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
+ 			tt.NotInHeap = sub.NotInHeap
+ 			typedef[name.Name] = &tt
+ 		}
++		globalMu.Unlock()
+ 
+ 		// If sub.Go.Name is "_Ctype_struct_foo" or "_Ctype_union_foo" or "_Ctype_class_foo",
+ 		// use that as the Go form for this typedef too, so that the typedef will be interchangeable
+@@ -2638,7 +2686,9 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
+ 
+ 			if isStructUnionClass(sub.Go) {
+ 				// Use the typedef name for C code.
++				globalMu.Lock()
+ 				typedef[sub.Go.(*ast.Ident).Name].C = t.C
++				globalMu.Unlock()
+ 			}
+ 
+ 			// If we've seen this typedef before, and it
+@@ -2698,8 +2748,12 @@ func (c *typeConv) loadType(dtype dwarf.Type, pos token.Pos, parent string) *Typ
+ 			}
+ 			s = strings.Replace(s, " ", "", -1)
+ 			name := c.Ident("_Ctype_" + s)
++
+ 			tt := *t
++			globalMu.Lock()
+ 			typedef[name.Name] = &tt
++			globalMu.Unlock()
++
+ 			if !*godefs {
+ 				t.Go = name
+ 			}
+diff --git a/src/cmd/cgo/godefs.go b/src/cmd/cgo/godefs.go
+index c0d59aee01d7..740aa3bc30b2 100644
+--- a/src/cmd/cgo/godefs.go
++++ b/src/cmd/cgo/godefs.go
+@@ -115,11 +115,11 @@ func (p *Package) godefs(f *File) string {
+ 	return buf.String()
+ }
+ 
+-var gofmtBuf bytes.Buffer
+-
+ // gofmt returns the gofmt-formatted string for an AST node.
+ func gofmt(n interface{}) string {
+-	gofmtBuf.Reset()
++	var gofmtBuf strings.Builder
++	gofmtBuf.Grow(512)
++
+ 	err := printer.Fprint(&gofmtBuf, fset, n)
+ 	if err != nil {
+ 		return "<" + err.Error() + ">"
+diff --git a/src/cmd/cgo/main.go b/src/cmd/cgo/main.go
+index 14642b7576b0..36e76908c602 100644
+--- a/src/cmd/cgo/main.go
++++ b/src/cmd/cgo/main.go
+@@ -26,6 +26,7 @@ import (
+ 	"runtime"
+ 	"sort"
+ 	"strings"
++	"sync"
+ 
+ 	"cmd/internal/edit"
+ 	"cmd/internal/objabi"
+@@ -33,6 +34,8 @@ import (
+ 
+ // A Package collects information about the package we're going to write.
+ type Package struct {
++	workerID int
++
+ 	PackageName string // name of package
+ 	PackagePath string
+ 	PtrSize     int64
+@@ -40,17 +43,36 @@ type Package struct {
+ 	GccOptions  []string
+ 	GccIsClang  bool
+ 	CgoFlags    map[string][]string // #cgo flags (CFLAGS, LDFLAGS)
+-	Written     map[string]bool
+-	Name        map[string]*Name // accumulated Name from Files
++	Written     *WrittenFiles
++	Name        map[string]*Name // TODO: parallelize. accumulated Name from Files
+ 	ExpFunc     []*ExpFunc       // accumulated ExpFunc from Files
+-	Decl        []ast.Decl
+-	GoFiles     []string        // list of Go files
+-	GccFiles    []string        // list of gcc output files
+-	Preamble    string          // collected preamble for _cgo_export.h
++	Decl        []ast.Decl       `json:"-"`
++	GoFiles     []string         // list of Go files
++	GccFiles    []string         // list of gcc output files
++	Preamble    string           // collected preamble for _cgo_export.h
++
+ 	typedefs    map[string]bool // type names that appear in the types of the objects we're interested in
+ 	typedefList []typedefInfo
+ }
+ 
++type WrittenFiles struct {
++	mut     sync.Mutex
++	written map[string]struct{}
++}
++
++func (w *WrittenFiles) Mark(file string) (shouldWrite bool) {
++	w.mut.Lock()
++	defer w.mut.Unlock()
++
++	_, already := w.written[file]
++	if already {
++		return false
++	}
++
++	w.written[file] = struct{}{}
++	return true
++}
++
+ // A typedefInfo is an element on Package.typedefList: a typedef name
+ // and the position where it was required.
+ type typedefInfo struct {
+@@ -87,7 +109,7 @@ func nameKeys(m map[string]*Name) []string {
+ 
+ // A Call refers to a call of a C.xxx function in the AST.
+ type Call struct {
+-	Call     *ast.CallExpr
++	Call     *ast.CallExpr `json:"-"`
+ 	Deferred bool
+ 	Done     bool
+ }
+@@ -95,7 +117,7 @@ type Call struct {
+ // A Ref refers to an expression of the form C.xxx in the AST.
+ type Ref struct {
+ 	Name    *Name
+-	Expr    *ast.Expr
++	Expr    *ast.Expr `json:"-"`
+ 	Context astContext
+ 	Done    bool
+ }
+@@ -108,13 +130,13 @@ var nameKinds = []string{"iconst", "fconst", "sconst", "type", "var", "fpvar", "
+ 
+ // A Name collects information about C.xxx.
+ type Name struct {
+-	Go       string // name used in Go referring to package C
+-	Mangle   string // name used in generated Go
+-	C        string // name used in C
+-	Define   string // #define expansion
+-	Kind     string // one of the nameKinds
+-	Type     *Type  // the type of xxx
+-	FuncType *FuncType
++	Go       string    // name used in Go referring to package C
++	Mangle   string    // name used in generated Go
++	C        string    // name used in C
++	Define   string    // #define expansion
++	Kind     string    // one of the nameKinds
++	Type     *Type     `json:"-"` // the type of xxx
++	FuncType *FuncType `json:"-"`
+ 	AddError bool
+ 	Const    string // constant definition
+ }
+@@ -133,23 +155,23 @@ func (n *Name) IsConst() bool {
+ // Such functions are identified in the Go input file
+ // by doc comments containing the line //export ExpName
+ type ExpFunc struct {
+-	Func    *ast.FuncDecl
+-	ExpName string // name to use from C
++	Func    *ast.FuncDecl `json:"-"`
++	ExpName string        // name to use from C
+ 	Doc     string
+ }
+ 
+ // A TypeRepr contains the string representation of a type.
+ type TypeRepr struct {
+ 	Repr       string
+-	FormatArgs []interface{}
++	FormatArgs []interface{} `json:"-"`
+ }
+ 
+ // A Type collects information about a type in both the C and Go worlds.
+ type Type struct {
+ 	Size       int64
+ 	Align      int64
+-	C          *TypeRepr
+-	Go         ast.Expr
++	C          *TypeRepr `json:"-"`
++	Go         ast.Expr  `json:"-"`
+ 	EnumValues map[string]int64
+ 	Typedef    string
+ 	BadPointer bool // this pointer type should be represented as a uintptr (deprecated)
+@@ -223,6 +245,8 @@ var cPrefix string
+ 
+ var fset = token.NewFileSet()
+ 
++var nparallel = flag.Int("nparallel", runtime.GOMAXPROCS(0), "maximum parallel jobs")
++
+ var dynobj = flag.String("dynimport", "", "if non-empty, print dynamic import data for that file")
+ var dynout = flag.String("dynout", "", "write -dynimport output to this file")
+ var dynpackage = flag.String("dynpackage", "main", "set Go package for -dynimport output")
+@@ -241,11 +265,13 @@ var exportHeader = flag.String("exportheader", "", "where to write export header
+ var gccgo = flag.Bool("gccgo", false, "generate files for use with gccgo")
+ var gccgoprefix = flag.String("gccgoprefix", "", "-fgo-prefix option used with gccgo")
+ var gccgopkgpath = flag.String("gccgopkgpath", "", "-fgo-pkgpath option used with gccgo")
+-var gccgoMangler func(string) string
+ var importRuntimeCgo = flag.Bool("import_runtime_cgo", true, "import runtime/cgo in generated code")
+ var importSyscall = flag.Bool("import_syscall", true, "import syscall in generated code")
+ var trimpath = flag.String("trimpath", "", "applies supplied rewrites or trims prefixes to recorded source file paths")
+ 
++var gccgoMangler func(string) string
++var gccgoManglerOnce sync.Once
++
+ var goarch, goos, gomips, gomips64 string
+ var gccBaseCmd []string
+ 
+@@ -302,7 +328,7 @@ func main() {
+ 		}
+ 	}
+ 
+-	p := newPackage(args[:i])
++	p := newPackage(args[:i], 0)
+ 
+ 	// We need a C compiler to be available. Check this.
+ 	var err error
+@@ -370,43 +396,152 @@ func main() {
+ 	}
+ 	*objDir += string(filepath.Separator)
+ 
+-	for i, input := range goFiles {
+-		f := fs[i]
+-		p.Translate(f)
+-		for _, cref := range f.Ref {
+-			switch cref.Context {
+-			case ctxCall, ctxCall2:
+-				if cref.Name.Kind != "type" {
+-					break
+-				}
+-				old := *cref.Expr
+-				*cref.Expr = cref.Name.Type.Go
+-				f.Edit.Replace(f.offset(old.Pos()), f.offset(old.End()), gofmt(cref.Name.Type.Go))
+-			}
+-		}
+-		if nerrors > 0 {
+-			os.Exit(2)
+-		}
+-		p.PackagePath = f.Package
+-		p.Record(f)
+-		if *godefs {
+-			os.Stdout.WriteString(p.godefs(f))
+-		} else {
+-			p.writeOutput(f, input)
++	pkgs := doFiles(p, goFiles, fs)
++
++	if !*godefs {
++		pkg := coalescePackages(pkgs)
++		pkg.writeDefs()
++	}
++
++	if nerrors.get() > 0 {
++		os.Exit(2)
++	}
++}
++
++type doFileJob struct {
++	input string
++	file  *File
++}
++
++func doFiles(p *Package, inputs []string, files []*File) []*Package {
++	if len(inputs) != len(files) {
++		fatalf("unexpected len(inputs) %d != len(files) %d", len(inputs), len(files))
++	}
++
++	jobCh := make(chan doFileJob)
++
++	var wg sync.WaitGroup
++	wg.Add(*nparallel)
++
++	pkgs := make([]*Package, *nparallel)
++	pkgs[0] = p
++
++	for n := 1; n < *nparallel; n++ {
++		pkgs[n] = cloneNewPackage(p, n)
++	}
++
++	for n := 0; n < *nparallel; n++ {
++		p := pkgs[n]
++		go func() {
++			doFileWorker(p, jobCh)
++			wg.Done()
++		}()
++	}
++
++	for i := range inputs {
++		jobCh <- doFileJob{
++			input: inputs[i],
++			file:  files[i],
+ 		}
+ 	}
+ 
+-	if !*godefs {
+-		p.writeDefs()
++	close(jobCh)
++	wg.Wait()
++
++	return pkgs
++}
++
++func doFileWorker(p *Package, jobCh <-chan doFileJob) *Package {
++	for job := range jobCh {
++		doFile(p, job.input, job.file)
+ 	}
+-	if nerrors > 0 {
++
++	return p
++}
++
++func doFile(p *Package, input string, f *File) {
++	p.Translate(f)
++	for _, cref := range f.Ref {
++		switch cref.Context {
++		case ctxCall, ctxCall2:
++			if cref.Name.Kind != "type" {
++				break
++			}
++			old := *cref.Expr
++			*cref.Expr = cref.Name.Type.Go
++			f.Edit.Replace(f.offset(old.Pos()), f.offset(old.End()), gofmt(cref.Name.Type.Go))
++		}
++	}
++	if nerrors.get() > 0 {
+ 		os.Exit(2)
+ 	}
++	p.PackagePath = f.Package
++	p.Record(f)
++	if *godefs {
++		os.Stdout.WriteString(p.godefs(f))
++	} else {
++		p.writeOutput(f, input)
++	}
++}
++
++func cloneNewPackage(p *Package, workerID int) *Package {
++	return &Package{
++		workerID:    workerID,
++		PackageName: p.PackageName,
++		PackagePath: p.PackagePath,
++		PtrSize:     p.PtrSize,
++		IntSize:     p.IntSize,
++		GccOptions:  p.GccOptions, // init in main only
++		GccIsClang:  p.GccIsClang,
++		CgoFlags:    p.CgoFlags, // init in main only
++		Written:     p.Written,
++	}
++}
++
++func coalescePackages(pkgs []*Package) *Package {
++	if len(pkgs) == 0 {
++		return nil
++	}
++
++	nonEmpty := pkgs[:0]
++	for _, p := range pkgs {
++		if p.PackageName != "" {
++			nonEmpty = append(nonEmpty, p)
++		}
++	}
++
++	pkgs = nonEmpty
++	pkg := pkgs[0]
++
++	for _, p := range pkgs[1:] {
++		pkg.PackageName = p.PackageName
++		pkg.PackagePath = p.PackagePath
++		pkg.ExpFunc = append(pkg.ExpFunc, p.ExpFunc...)
++		pkg.Decl = append(pkg.Decl, p.Decl...)
++		pkg.GoFiles = append(pkg.GoFiles, p.GoFiles...)
++		pkg.GccFiles = append(pkg.GccFiles, p.GccFiles...)
++		pkg.Preamble += p.Preamble
++
++		if pkg.Name == nil && p.Name != nil {
++			pkg.Name = p.Name
++			continue
++		}
++
++		for k, v := range p.Name {
++			if o, ok := pkg.Name[k]; ok && !reflect.DeepEqual(v, o) {
++				fatalf("duplicate pkg.Name %s, %v == %v", k, v, o)
++			}
++
++			pkg.Name[k] = v
++		}
++	}
++
++	return pkg
+ }
+ 
+ // newPackage returns a new Package that will invoke
+ // gcc with the additional arguments specified in args.
+-func newPackage(args []string) *Package {
++func newPackage(args []string, workerID int) *Package {
+ 	goarch = runtime.GOARCH
+ 	if s := os.Getenv("GOARCH"); s != "" {
+ 		goarch = s
+@@ -432,10 +567,13 @@ func newPackage(args []string) *Package {
+ 	os.Setenv("LC_ALL", "C")
+ 
+ 	p := &Package{
++		workerID: workerID,
+ 		PtrSize:  ptrSize,
+ 		IntSize:  intSize,
+ 		CgoFlags: make(map[string][]string),
+-		Written:  make(map[string]bool),
++		Written: &WrittenFiles{
++			written: make(map[string]struct{}),
++		},
+ 	}
+ 	p.addToFlag("CFLAGS", args)
+ 	return p
+diff --git a/src/cmd/cgo/out.go b/src/cmd/cgo/out.go
+index 4968f7059d9b..b934d12d9423 100644
+--- a/src/cmd/cgo/out.go
++++ b/src/cmd/cgo/out.go
+@@ -32,14 +32,13 @@ var (
+ 
+ // writeDefs creates output files to be compiled by gc and gcc.
+ func (p *Package) writeDefs() {
+-	var fgo2, fc io.Writer
+-	f := creat(*objDir + "_cgo_gotypes.go")
+-	defer f.Close()
+-	fgo2 = f
++	fgo2 := creat(*objDir + "_cgo_gotypes.go")
++	defer fgo2.Close()
++
++	var fc *os.File
+ 	if *gccgo {
+-		f := creat(*objDir + "_cgo_defun.c")
+-		defer f.Close()
+-		fc = f
++		fc = creat(*objDir + "_cgo_defun.c")
++		defer fc.Close()
+ 	}
+ 	fm := creat(*objDir + "_cgo_main.c")
+ 
+@@ -682,12 +681,11 @@ var isBuiltin = map[string]bool{
+ 
+ func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) {
+ 	name := n.Mangle
+-	if isBuiltin[name] || p.Written[name] {
++	if isBuiltin[name] || !p.Written.Mark(name) {
+ 		// The builtins are already defined in the C prolog, and we don't
+ 		// want to duplicate function definitions we've already done.
+ 		return
+ 	}
+-	p.Written[name] = true
+ 
+ 	if *gccgo {
+ 		p.writeGccgoOutputFunc(fgcc, n)
+@@ -1294,7 +1292,7 @@ func (p *Package) writeExportHeader(fgcch io.Writer) {
+ 
+ // gccgoToSymbol converts a name to a mangled symbol for gccgo.
+ func gccgoToSymbol(ppath string) string {
+-	if gccgoMangler == nil {
++	gccgoManglerOnce.Do(func() {
+ 		var err error
+ 		cmd := os.Getenv("GCCGO")
+ 		if cmd == "" {
+@@ -1307,7 +1305,7 @@ func gccgoToSymbol(ppath string) string {
+ 		if err != nil {
+ 			fatalf("%v", err)
+ 		}
+-	}
++	})
+ 	return gccgoMangler(ppath)
+ }
+ 
+diff --git a/src/cmd/cgo/util.go b/src/cmd/cgo/util.go
+index 00d931b98a0c..bea77b841466 100644
+--- a/src/cmd/cgo/util.go
++++ b/src/cmd/cgo/util.go
+@@ -11,6 +11,7 @@ import (
+ 	exec "internal/execabs"
+ 	"io/ioutil"
+ 	"os"
++	"sync/atomic"
+ )
+ 
+ // run runs the command argv, feeding in stdin on standard input.
+@@ -87,16 +88,22 @@ func lineno(pos token.Pos) string {
+ func fatalf(msg string, args ...interface{}) {
+ 	// If we've already printed other errors, they might have
+ 	// caused the fatal condition. Assume they're enough.
+-	if nerrors == 0 {
++	if !nerrors.has() {
+ 		fmt.Fprintf(os.Stderr, "cgo: "+msg+"\n", args...)
+ 	}
+ 	os.Exit(2)
+ }
+ 
+-var nerrors int
++type atomicNErrors uint64
++
++var nerrors atomicNErrors
++
++func (n *atomicNErrors) add()        { atomic.AddUint64((*uint64)(n), 1) }
++func (n *atomicNErrors) has() bool   { return n.get() > 0 }
++func (n *atomicNErrors) get() uint64 { return atomic.LoadUint64((*uint64)(n)) }
+ 
+ func error_(pos token.Pos, msg string, args ...interface{}) {
+-	nerrors++
++	nerrors.add()
+ 	if pos.IsValid() {
+ 		fmt.Fprintf(os.Stderr, "%s: ", fset.Position(pos).String())
+ 	} else {
+diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go
+index 0af39a991c02..ac658f4803ef 100644
+--- a/src/cmd/go/internal/work/exec.go
++++ b/src/cmd/go/internal/work/exec.go
+@@ -2821,7 +2821,12 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo
+ 		cgoflags = append(cgoflags, "-trimpath", strings.Join(trimpath, ";"))
+ 	}
+ 
+-	if err := b.run(a, execdir, p.ImportPath, cgoenv, cfg.BuildToolexec, cgoExe, "-objdir", objdir, "-importpath", p.ImportPath, cgoflags, "--", cgoCPPFLAGS, cgoCFLAGS, cgofiles); err != nil {
++	nparallel := runtime.GOMAXPROCS(0)
++	if nparallel > 4 {
++		nparallel = 4
++	}
++
++	if err := b.run(a, execdir, p.ImportPath, cgoenv, cfg.BuildToolexec, cgoExe, "-nparallel", strconv.Itoa(nparallel), "-objdir", objdir, "-importpath", p.ImportPath, cgoflags, "--", cgoCPPFLAGS, cgoCFLAGS, cgofiles); err != nil {
+ 		return nil, nil, err
+ 	}
+ 	outGo = append(outGo, gofiles...)

patches/diamondburned/go/ec3e1c9471f170187b6a7c83ab0364253f895c28.patch

@@ -0,0 +1,159 @@
+From ec3e1c9471f170187b6a7c83ab0364253f895c28 Mon Sep 17 00:00:00 2001
+From: diamondburned <datutbrus@gmail.com>
+Date: Thu, 5 Aug 2021 16:43:01 -0700
+Subject: [PATCH] cmd/go/internal/work: concurrent ccompile routines
+
+This commit allows execution of gcc and others after the cgo stage to be
+concurrent.
+
+Prior to this change, only 1 gcc instance will run at a time.
+
+After this, these instances will be launched off in multiples at the
+same time, where the parallelism is determined by the thread count, but
+capped at 4 using gcBackendConcurrency to calculate.
+---
+ src/cmd/go/internal/work/exec.go | 97 +++++++++++++++++++++++++++-----
+ 1 file changed, 82 insertions(+), 15 deletions(-)
+
+diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go
+index ac80f503cd89..0af39a991c02 100644
+--- a/src/cmd/go/internal/work/exec.go
++++ b/src/cmd/go/internal/work/exec.go
+@@ -2838,48 +2838,70 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo
+ 		return objdir + fmt.Sprintf("_x%03d.o", oseq)
+ 	}
+ 
++	jobCh := make(chan func() error)
++	jobErrCh := make(chan error)
++
++	nparallel = gccConcurrency(a)
++	go func() { jobErrCh <- runJobDispatcher(nparallel, jobCh) }()
++
+ 	// gcc
+ 	cflags := str.StringList(cgoCPPFLAGS, cgoCFLAGS)
+ 	for _, cfile := range cfiles {
+ 		ofile := nextOfile()
+-		if err := b.gcc(a, p, a.Objdir, ofile, cflags, objdir+cfile); err != nil {
+-			return nil, nil, err
+-		}
++		cfile := cfile
+ 		outObj = append(outObj, ofile)
++
++		jobCh <- func() error {
++			return b.gcc(a, p, a.Objdir, ofile, cflags, objdir+cfile)
++		}
+ 	}
+ 
+ 	for _, file := range gccfiles {
+ 		ofile := nextOfile()
+-		if err := b.gcc(a, p, a.Objdir, ofile, cflags, file); err != nil {
+-			return nil, nil, err
+-		}
++		ifile := file
+ 		outObj = append(outObj, ofile)
++
++		jobCh <- func() error {
++			return b.gcc(a, p, a.Objdir, ofile, cflags, ifile)
++		}
+ 	}
+ 
+ 	cxxflags := str.StringList(cgoCPPFLAGS, cgoCXXFLAGS)
+ 	for _, file := range gxxfiles {
+ 		ofile := nextOfile()
+-		if err := b.gxx(a, p, a.Objdir, ofile, cxxflags, file); err != nil {
+-			return nil, nil, err
+-		}
++		ifile := file
+ 		outObj = append(outObj, ofile)
++
++		jobCh <- func() error {
++			return b.gxx(a, p, a.Objdir, ofile, cxxflags, ifile)
++		}
+ 	}
+ 
+ 	for _, file := range mfiles {
+ 		ofile := nextOfile()
+-		if err := b.gcc(a, p, a.Objdir, ofile, cflags, file); err != nil {
+-			return nil, nil, err
+-		}
++		ifile := file
+ 		outObj = append(outObj, ofile)
++
++		jobCh <- func() error {
++			return b.gcc(a, p, a.Objdir, ofile, cflags, ifile)
++		}
+ 	}
+ 
+ 	fflags := str.StringList(cgoCPPFLAGS, cgoFFLAGS)
+ 	for _, file := range ffiles {
+ 		ofile := nextOfile()
+-		if err := b.gfortran(a, p, a.Objdir, ofile, fflags, file); err != nil {
+-			return nil, nil, err
+-		}
++		ifile := file
+ 		outObj = append(outObj, ofile)
++
++		jobCh <- func() error {
++			return b.gfortran(a, p, a.Objdir, ofile, fflags, ifile)
++		}
++	}
++
++	close(jobCh)
++
++	if err := <-jobErrCh; err != nil {
++		return nil, nil, err
+ 	}
+ 
+ 	switch cfg.BuildToolchainName {
+@@ -2980,6 +3002,51 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo
+ 	return outGo, outObj, nil
+ }
+ 
++// gccConcurrency returns the concurrency level for spawning gcc processes.
++func gccConcurrency(a *Action) int {
++	gcflags := str.StringList(forcedGcflags, a.Package.Internal.Gcflags)
++	return gcBackendConcurrency(gcflags)
++}
++
++func runJobDispatcher(nparallel int, jobCh <-chan func() error) error {
++	var wg sync.WaitGroup
++
++	var firstErr error
++	var errNum int
++	var errMut sync.Mutex
++
++	semaphore := make(chan struct{}, nparallel)
++
++	for job := range jobCh {
++		job := job
++
++		wg.Add(1)
++		semaphore <- struct{}{}
++
++		go func() {
++			defer wg.Done()
++			defer func() { <-semaphore }()
++
++			if err := job(); err != nil {
++				errMut.Lock()
++				if firstErr == nil {
++					firstErr = err
++				}
++				errNum++
++				errMut.Unlock()
++			}
++		}()
++	}
++
++	wg.Wait()
++
++	if errNum > 0 {
++		return fmt.Errorf("encountered %d errors, including: %w", errNum, firstErr)
++	}
++
++	return nil
++}
++
+ // dynimport creates a Go source file named importGo containing
+ // //go:cgo_import_dynamic directives for each symbol or library
+ // dynamically imported by the object files outObj.