math-optim/patches/diamondburned/go/ec3e1c9471f170187b6a7c83ab0364253f895c28.patch

From ec3e1c9471f170187b6a7c83ab0364253f895c28 Mon Sep 17 00:00:00 2001
From: diamondburned <datutbrus@gmail.com>
Date: Thu, 5 Aug 2021 16:43:01 -0700
Subject: [PATCH] cmd/go/internal/work: concurrent ccompile routines

This commit allows execution of gcc and others after the cgo stage to be
concurrent.

Prior to this change, only 1 gcc instance will run at a time.

After this, these instances will be launched off in multiples at the
same time, where the parallelism is determined by the thread count, but
capped at 4 using gcBackendConcurrency to calculate.
---
 src/cmd/go/internal/work/exec.go | 97 +++++++++++++++++++++++++++-----
 1 file changed, 82 insertions(+), 15 deletions(-)

diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go
index ac80f503cd89..0af39a991c02 100644
--- a/src/cmd/go/internal/work/exec.go
+++ b/src/cmd/go/internal/work/exec.go
@@ -2838,48 +2838,70 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo
 		return objdir + fmt.Sprintf("_x%03d.o", oseq)
 	}
 
+	jobCh := make(chan func() error)
+	jobErrCh := make(chan error)
+
+	nparallel = gccConcurrency(a)
+	go func() { jobErrCh <- runJobDispatcher(nparallel, jobCh) }()
+
 	// gcc
 	cflags := str.StringList(cgoCPPFLAGS, cgoCFLAGS)
 	for _, cfile := range cfiles {
 		ofile := nextOfile()
-		if err := b.gcc(a, p, a.Objdir, ofile, cflags, objdir+cfile); err != nil {
-			return nil, nil, err
-		}
+		cfile := cfile
 		outObj = append(outObj, ofile)
+
+		jobCh <- func() error {
+			return b.gcc(a, p, a.Objdir, ofile, cflags, objdir+cfile)
+		}
 	}
 
 	for _, file := range gccfiles {
 		ofile := nextOfile()
-		if err := b.gcc(a, p, a.Objdir, ofile, cflags, file); err != nil {
-			return nil, nil, err
-		}
+		ifile := file
 		outObj = append(outObj, ofile)
+
+		jobCh <- func() error {
+			return b.gcc(a, p, a.Objdir, ofile, cflags, ifile)
+		}
 	}
 
 	cxxflags := str.StringList(cgoCPPFLAGS, cgoCXXFLAGS)
 	for _, file := range gxxfiles {
 		ofile := nextOfile()
-		if err := b.gxx(a, p, a.Objdir, ofile, cxxflags, file); err != nil {
-			return nil, nil, err
-		}
+		ifile := file
 		outObj = append(outObj, ofile)
+
+		jobCh <- func() error {
+			return b.gxx(a, p, a.Objdir, ofile, cxxflags, ifile)
+		}
 	}
 
 	for _, file := range mfiles {
 		ofile := nextOfile()
-		if err := b.gcc(a, p, a.Objdir, ofile, cflags, file); err != nil {
-			return nil, nil, err
-		}
+		ifile := file
 		outObj = append(outObj, ofile)
+
+		jobCh <- func() error {
+			return b.gcc(a, p, a.Objdir, ofile, cflags, ifile)
+		}
 	}
 
 	fflags := str.StringList(cgoCPPFLAGS, cgoFFLAGS)
 	for _, file := range ffiles {
 		ofile := nextOfile()
-		if err := b.gfortran(a, p, a.Objdir, ofile, fflags, file); err != nil {
-			return nil, nil, err
-		}
+		ifile := file
 		outObj = append(outObj, ofile)
+
+		jobCh <- func() error {
+			return b.gfortran(a, p, a.Objdir, ofile, fflags, ifile)
+		}
+	}
+
+	close(jobCh)
+
+	if err := <-jobErrCh; err != nil {
+		return nil, nil, err
 	}
 
 	switch cfg.BuildToolchainName {
@@ -2980,6 +3002,51 @@ func (b *Builder) cgo(a *Action, cgoExe, objdir string, pcCFLAGS, pcLDFLAGS, cgo
 	return outGo, outObj, nil
 }
 
+// gccConcurrency returns the concurrency level for spawning gcc processes.
+func gccConcurrency(a *Action) int {
+	gcflags := str.StringList(forcedGcflags, a.Package.Internal.Gcflags)
+	return gcBackendConcurrency(gcflags)
+}
+
+func runJobDispatcher(nparallel int, jobCh <-chan func() error) error {
+	var wg sync.WaitGroup
+
+	var firstErr error
+	var errNum int
+	var errMut sync.Mutex
+
+	semaphore := make(chan struct{}, nparallel)
+
+	for job := range jobCh {
+		job := job
+
+		wg.Add(1)
+		semaphore <- struct{}{}
+
+		go func() {
+			defer wg.Done()
+			defer func() { <-semaphore }()
+
+			if err := job(); err != nil {
+				errMut.Lock()
+				if firstErr == nil {
+					firstErr = err
+				}
+				errNum++
+				errMut.Unlock()
+			}
+		}()
+	}
+
+	wg.Wait()
+
+	if errNum > 0 {
+		return fmt.Errorf("encountered %d errors, including: %w", errNum, firstErr)
+	}
+
+	return nil
+}
+
 // dynimport creates a Go source file named importGo containing
 // //go:cgo_import_dynamic directives for each symbol or library
 // dynamically imported by the object files outObj.