math-optim/patches/diamondburned/go/ec3e1c9471f170187b6a7c83ab0364253f895c28.patch

160 lines
4.2 KiB
Diff
Raw Normal View History

2022-06-14 16:46:11 +02:00
From ec3e1c9471f170187b6a7c83ab0364253f895c28 Mon Sep 17 00:00:00 2001
From: diamondburned <datutbrus@gmail.com>
Date: Thu, 5 Aug 2021 16:43:01 -0700
Subject: [PATCH] cmd/go/internal/work: concurrent ccompile routines
This commit allows execution of gcc and others after the cgo stage to be
concurrent.
Prior to this change, only 1 gcc instance will run at a time.
After this, these instances will be launched off in multiples at the
same time, where the parallelism is determined by the thread count, but
capped at 4 using gcBackendConcurrency to calculate.
---
src/cmd/go/internal/work/exec.go | 97 +++++++++++++++++++++++++++-----
1 file changed, 82 insertions(+), 15 deletions(-)
diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go
index ac80f503cd89..0af39a991c02 100644
--- a/src/cmd/go/internal/work/exec.go
+++ b/src/cmd/go/internal/work/exec.go
@@ -2838,48 +2838,70 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo
return objdir + fmt.Sprintf("_x%03d.o", oseq)
}
+ jobCh := make(chan func() error)
+ jobErrCh := make(chan error)
+
+ nparallel = gccConcurrency(a)
+ go func() { jobErrCh <- runJobDispatcher(nparallel, jobCh) }()
+
// gcc
cflags := str.StringList(cgoCPPFLAGS, cgoCFLAGS)
for _, cfile := range cfiles {
ofile := nextOfile()
- if err := b.gcc(a, p, a.Objdir, ofile, cflags, objdir+cfile); err != nil {
- return nil, nil, err
- }
+ cfile := cfile
outObj = append(outObj, ofile)
+
+ jobCh <- func() error {
+ return b.gcc(a, p, a.Objdir, ofile, cflags, objdir+cfile)
+ }
}
for _, file := range gccfiles {
ofile := nextOfile()
- if err := b.gcc(a, p, a.Objdir, ofile, cflags, file); err != nil {
- return nil, nil, err
- }
+ ifile := file
outObj = append(outObj, ofile)
+
+ jobCh <- func() error {
+ return b.gcc(a, p, a.Objdir, ofile, cflags, ifile)
+ }
}
cxxflags := str.StringList(cgoCPPFLAGS, cgoCXXFLAGS)
for _, file := range gxxfiles {
ofile := nextOfile()
- if err := b.gxx(a, p, a.Objdir, ofile, cxxflags, file); err != nil {
- return nil, nil, err
- }
+ ifile := file
outObj = append(outObj, ofile)
+
+ jobCh <- func() error {
+ return b.gxx(a, p, a.Objdir, ofile, cxxflags, ifile)
+ }
}
for _, file := range mfiles {
ofile := nextOfile()
- if err := b.gcc(a, p, a.Objdir, ofile, cflags, file); err != nil {
- return nil, nil, err
- }
+ ifile := file
outObj = append(outObj, ofile)
+
+ jobCh <- func() error {
+ return b.gcc(a, p, a.Objdir, ofile, cflags, ifile)
+ }
}
fflags := str.StringList(cgoCPPFLAGS, cgoFFLAGS)
for _, file := range ffiles {
ofile := nextOfile()
- if err := b.gfortran(a, p, a.Objdir, ofile, fflags, file); err != nil {
- return nil, nil, err
- }
+ ifile := file
outObj = append(outObj, ofile)
+
+ jobCh <- func() error {
+ return b.gfortran(a, p, a.Objdir, ofile, fflags, ifile)
+ }
+ }
+
+ close(jobCh)
+
+ if err := <-jobErrCh; err != nil {
+ return nil, nil, err
}
switch cfg.BuildToolchainName {
@@ -2980,6 +3002,51 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo
return outGo, outObj, nil
}
+// gccConcurrency returns the concurrency level for spawning gcc processes.
+func gccConcurrency(a *Action) int {
+ gcflags := str.StringList(forcedGcflags, a.Package.Internal.Gcflags)
+ return gcBackendConcurrency(gcflags)
+}
+
+func runJobDispatcher(nparallel int, jobCh <-chan func() error) error {
+ var wg sync.WaitGroup
+
+ var firstErr error
+ var errNum int
+ var errMut sync.Mutex
+
+ semaphore := make(chan struct{}, nparallel)
+
+ for job := range jobCh {
+ job := job
+
+ wg.Add(1)
+ semaphore <- struct{}{}
+
+ go func() {
+ defer wg.Done()
+ defer func() { <-semaphore }()
+
+ if err := job(); err != nil {
+ errMut.Lock()
+ if firstErr == nil {
+ firstErr = err
+ }
+ errNum++
+ errMut.Unlock()
+ }
+ }()
+ }
+
+ wg.Wait()
+
+ if errNum > 0 {
+ return fmt.Errorf("encountered %d errors, including: %w", errNum, firstErr)
+ }
+
+ return nil
+}
+
// dynimport creates a Go source file named importGo containing
// //go:cgo_import_dynamic directives for each symbol or library
// dynamically imported by the object files outObj.