c90f73f816
* guix/build/go-build-system.com (setup-go-environment): Set GOCACHE to a location within the build directory. Union "/var/cache/go/build" input directories to initialize the cache. Generate "trim.txt" within the cache, with the current time. Signed-off-by: Leo Famulari <leo@famulari.name>
377 lines
17 KiB
Scheme
377 lines
17 KiB
Scheme
;;; GNU Guix --- Functional package management for GNU
|
|
;;; Copyright © 2016 Petter <petter@mykolab.ch>
|
|
;;; Copyright © 2017, 2019 Leo Famulari <leo@famulari.name>
|
|
;;; Copyright © 2019 Maxim Cournoyer <maxim.cournoyer@gmail.com>
|
|
;;; Copyright © 2020 Jack Hill <jackhill@jackhill.us>
|
|
;;; Copyright © 2020 Jakub Kądziołka <kuba@kadziolka.net>
|
|
;;; Copyright © 2020, 2021 Efraim Flashner <efraim@flashner.co.il>
|
|
;;; Copyright © 2021 Sarah Morgensen <iskarian@mgsn.dev>
|
|
;;;
|
|
;;; This file is part of GNU Guix.
|
|
;;;
|
|
;;; GNU Guix is free software; you can redistribute it and/or modify it
|
|
;;; under the terms of the GNU General Public License as published by
|
|
;;; the Free Software Foundation; either version 3 of the License, or (at
|
|
;;; your option) any later version.
|
|
;;;
|
|
;;; GNU Guix is distributed in the hope that it will be useful, but
|
|
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
;;; GNU General Public License for more details.
|
|
;;;
|
|
;;; You should have received a copy of the GNU General Public License
|
|
;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
(define-module (guix build go-build-system)
|
|
#:use-module ((guix build gnu-build-system) #:prefix gnu:)
|
|
#:use-module (guix build union)
|
|
#:use-module (guix build utils)
|
|
#:use-module (ice-9 match)
|
|
#:use-module (ice-9 ftw)
|
|
#:use-module (srfi srfi-1)
|
|
#:use-module (rnrs io ports)
|
|
#:use-module (rnrs bytevectors)
|
|
#:export (%standard-phases
|
|
go-build))
|
|
|
|
;; Commentary:
|
|
;;
|
|
;; Build procedures for Go packages. This is the builder-side code.
|
|
;;
|
|
;; Software written in Go is either a 'package' (i.e. library) or 'command'
|
|
;; (i.e. executable). Both types can be built with either the `go build` or `go
|
|
;; install` commands. However, `go build` discards the result of the build
|
|
;; process for Go libraries, so we use `go install`, which preserves the
|
|
;; results. [0]
|
|
|
|
;; Go software is developed and built within a particular file system hierarchy
|
|
;; structure called a 'workspace' [1]. This workspace can be found by Go via
|
|
;; the GOPATH environment variable. Typically, all Go source code and compiled
|
|
;; objects are kept in a single workspace, but GOPATH may be a list of
|
|
;; directories [2]. In this go-build-system we create a file system union of
|
|
;; the Go-language dependencies. Previously, we made GOPATH a list of store
|
|
;; directories, but stopped because Go programs started keeping references to
|
|
;; these directories in Go 1.11:
|
|
;; <https://bugs.gnu.org/33620>.
|
|
;;
|
|
;; Go software, whether a package or a command, is uniquely named using an
|
|
;; 'import path'. The import path is based on the URL of the software's source.
|
|
;; Because most source code is provided over the internet, the import path is
|
|
;; typically a combination of the remote URL and the source repository's file
|
|
;; system structure. For example, the Go port of the common `du` command is
|
|
;; hosted on github.com, at <https://github.com/calmh/du>. Thus, the import
|
|
;; path is <github.com/calmh/du>. [3]
|
|
;;
|
|
;; It may be possible to automatically guess a package's import path based on
|
|
;; the source URL, but we don't try that in this revision of the
|
|
;; go-build-system.
|
|
;;
|
|
;; Modules of modular Go libraries are named uniquely with their
|
|
;; file system paths. For example, the supplemental but "standardized"
|
|
;; libraries developed by the Go upstream developers are available at
|
|
;; <https://golang.org/x/{net,text,crypto, et cetera}>. The Go IPv4
|
|
;; library's import path is <golang.org/x/net/ipv4>. The source of
|
|
;; such modular libraries must be unpacked at the top-level of the
|
|
;; file system structure of the library. So the IPv4 library should be
|
|
;; unpacked to <golang.org/x/net>. This is handled in the
|
|
;; go-build-system with the optional #:unpack-path key.
|
|
;;
|
|
;; In general, Go software is built using a standardized build mechanism
|
|
;; that does not require any build scripts like Makefiles. This means
|
|
;; that all modules of modular libraries cannot be built with a single
|
|
;; command. Each module must be built individually. This complicates
|
|
;; certain cases, and these issues are currently resolved by creating a
|
|
;; file system union of the required modules of such libraries. I think
|
|
;; this could be improved in future revisions of the go-build-system.
|
|
;;
|
|
;; TODO:
|
|
;; * Avoid copying dependencies into the build environment and / or avoid using
|
|
;; a tmpdir when creating the inputs union.
|
|
;; * Use Go modules [4]
|
|
;; * Re-use compiled packages [5]
|
|
;; * Avoid the go-inputs hack
|
|
;; * Stop needing remove-go-references (-trimpath ? )
|
|
;; * Remove module packages, only offering the full Git repos? This is
|
|
;; more idiomatic, I think, because Go downloads Git repos, not modules.
|
|
;; What are the trade-offs?
|
|
;;
|
|
;; [0] `go build`:
|
|
;; https://golang.org/cmd/go/#hdr-Compile_packages_and_dependencies
|
|
;; `go install`:
|
|
;; https://golang.org/cmd/go/#hdr-Compile_and_install_packages_and_dependencies
|
|
;; [1] Go workspace example, from <https://golang.org/doc/code.html#Workspaces>:
|
|
;; bin/
|
|
;; hello # command executable
|
|
;; outyet # command executable
|
|
;; pkg/
|
|
;; linux_amd64/
|
|
;; github.com/golang/example/
|
|
;; stringutil.a # package object
|
|
;; src/
|
|
;; github.com/golang/example/
|
|
;; .git/ # Git repository metadata
|
|
;; hello/
|
|
;; hello.go # command source
|
|
;; outyet/
|
|
;; main.go # command source
|
|
;; main_test.go # test source
|
|
;; stringutil/
|
|
;; reverse.go # package source
|
|
;; reverse_test.go # test source
|
|
;; golang.org/x/image/
|
|
;; .git/ # Git repository metadata
|
|
;; bmp/
|
|
;; reader.go # package source
|
|
;; writer.go # package source
|
|
;; ... (many more repositories and packages omitted) ...
|
|
;;
|
|
;; [2] https://golang.org/doc/code.html#GOPATH
|
|
;; [3] https://golang.org/doc/code.html#ImportPaths
|
|
;; [4] https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more
|
|
;; [5] https://bugs.gnu.org/32919
|
|
;;
|
|
;; Code:
|
|
|
|
(define* (setup-go-environment #:key inputs outputs goos goarch #:allow-other-keys)
|
|
"Prepare a Go build environment for INPUTS and OUTPUTS. Build a file system
|
|
union of INPUTS. Export GOPATH, which helps the compiler find the source code
|
|
of the package being built and its dependencies, and GOBIN, which determines
|
|
where executables (\"commands\") are installed to. This phase is sometimes used
|
|
by packages that use (guix build-system gnu) but have a handful of Go
|
|
dependencies, so it should be self-contained."
|
|
(define (search-input-directories dir)
|
|
(filter directory-exists?
|
|
(map (match-lambda
|
|
((name . directory)
|
|
(string-append directory "/" dir)))
|
|
inputs)))
|
|
|
|
;; Seed the Go build cache with the build caches from input packages.
|
|
(let ((cache (string-append (getcwd) "/go-build")))
|
|
(setenv "GOCACHE" cache)
|
|
(union-build cache
|
|
(search-input-directories "/var/cache/go/build")
|
|
;; Creating all directories isn't that bad, because there are
|
|
;; only ever 256 of them.
|
|
#:create-all-directories? #t
|
|
#:log-port (%make-void-port "w"))
|
|
|
|
;; Tell Go that the cache was recently trimmed, so it doesn't try to.
|
|
(call-with-output-file (string-append cache "/trim.txt")
|
|
(lambda (port)
|
|
(format port "~a" (current-time)))))
|
|
|
|
;; Using the current working directory as GOPATH makes it easier for packagers
|
|
;; who need to manipulate the unpacked source code.
|
|
(setenv "GOPATH" (getcwd))
|
|
;; Go 1.13 uses go modules by default. The go build system does not
|
|
;; currently support modules, so turn modules off to continue using the old
|
|
;; GOPATH behavior.
|
|
(setenv "GO111MODULE" "off")
|
|
(setenv "GOBIN" (string-append (assoc-ref outputs "out") "/bin"))
|
|
|
|
;; Make sure we're building for the correct architecture and OS targets
|
|
;; that Guix targets.
|
|
(setenv "GOARCH" (or goarch
|
|
(getenv "GOHOSTARCH")))
|
|
(setenv "GOOS" (or goos
|
|
(getenv "GOHOSTOS")))
|
|
(match goarch
|
|
("arm"
|
|
(setenv "GOARM" "7"))
|
|
((or "mips" "mipsel")
|
|
(setenv "GOMIPS" "hardfloat"))
|
|
((or "mips64" "mips64le")
|
|
(setenv "GOMIPS64" "hardfloat"))
|
|
((or "ppc64" "ppc64le")
|
|
(setenv "GOPPC64" "power8"))
|
|
(_ #t))
|
|
|
|
(let ((tmpdir (tmpnam)))
|
|
(match (go-inputs inputs)
|
|
(((names . directories) ...)
|
|
(union-build tmpdir (filter directory-exists? directories)
|
|
#:create-all-directories? #t
|
|
#:log-port (%make-void-port "w"))))
|
|
;; XXX A little dance because (guix build union) doesn't use mkdir-p.
|
|
(copy-recursively tmpdir
|
|
(string-append (getenv "GOPATH"))
|
|
#:keep-mtime? #t)
|
|
(delete-file-recursively tmpdir))
|
|
#t)
|
|
|
|
(define* (unpack #:key source import-path unpack-path #:allow-other-keys)
|
|
"Relative to $GOPATH, unpack SOURCE in UNPACK-PATH, or IMPORT-PATH when
|
|
UNPACK-PATH is unset. If the SOURCE archive has a single top level directory,
|
|
it is stripped so that the sources appear directly under UNPACK-PATH. When
|
|
SOURCE is a directory, copy its content into UNPACK-PATH instead of
|
|
unpacking."
|
|
(define (unpack-maybe-strip source dest)
|
|
(let* ((scratch-dir (string-append (or (getenv "TMPDIR") "/tmp")
|
|
"/scratch-dir"))
|
|
(out (mkdir-p scratch-dir)))
|
|
(with-directory-excursion scratch-dir
|
|
(if (string-suffix? ".zip" source)
|
|
(invoke "unzip" source)
|
|
(invoke "tar" "-xvf" source))
|
|
(let ((top-level-files (remove (lambda (x)
|
|
(member x '("." "..")))
|
|
(scandir "."))))
|
|
(match top-level-files
|
|
((top-level-file)
|
|
(when (file-is-directory? top-level-file)
|
|
(copy-recursively top-level-file dest #:keep-mtime? #t)))
|
|
(_
|
|
(copy-recursively "." dest #:keep-mtime? #t)))))
|
|
(delete-file-recursively scratch-dir)))
|
|
|
|
(when (string-null? import-path)
|
|
(display "WARNING: The Go import path is unset.\n"))
|
|
(when (string-null? unpack-path)
|
|
(set! unpack-path import-path))
|
|
(let ((dest (string-append (getenv "GOPATH") "/src/" unpack-path)))
|
|
(mkdir-p dest)
|
|
(if (file-is-directory? source)
|
|
(copy-recursively source dest #:keep-mtime? #t)
|
|
(unpack-maybe-strip source dest)))
|
|
#t)
|
|
|
|
(define (go-package? name)
|
|
(string-prefix? "go-" name))
|
|
|
|
(define (go-inputs inputs)
|
|
"Return the alist of INPUTS that are Go software."
|
|
;; XXX This should not check the file name of the store item. Instead we
|
|
;; should pass, from the host side, the list of inputs that are packages using
|
|
;; the go-build-system.
|
|
(alist-delete "go" ; Exclude the Go compiler
|
|
(alist-delete "source" ; Exclude the source code of the package being built
|
|
(filter (match-lambda
|
|
((label . directory)
|
|
(go-package? ((compose package-name->name+version
|
|
strip-store-file-name)
|
|
directory)))
|
|
(_ #f))
|
|
inputs))))
|
|
|
|
(define* (build #:key import-path build-flags #:allow-other-keys)
|
|
"Build the package named by IMPORT-PATH."
|
|
(with-throw-handler
|
|
#t
|
|
(lambda _
|
|
(apply invoke "go" "install"
|
|
"-v" ; print the name of packages as they are compiled
|
|
"-x" ; print each command as it is invoked
|
|
;; Respectively, strip the symbol table and debug
|
|
;; information, and the DWARF symbol table.
|
|
"-ldflags=-s -w"
|
|
`(,@build-flags ,import-path)))
|
|
(lambda (key . args)
|
|
(display (string-append "Building '" import-path "' failed.\n"
|
|
"Here are the results of `go env`:\n"))
|
|
(invoke "go" "env"))))
|
|
|
|
;; Can this also install commands???
|
|
(define* (check #:key tests? import-path #:allow-other-keys)
|
|
"Run the tests for the package named by IMPORT-PATH."
|
|
(when tests?
|
|
(invoke "go" "test" import-path))
|
|
#t)
|
|
|
|
(define* (install #:key install-source? outputs import-path unpack-path #:allow-other-keys)
|
|
"Install the source code of IMPORT-PATH to the primary output directory.
|
|
Compiled executable files (Go \"commands\") should have already been installed
|
|
to the store based on $GOBIN in the build phase.
|
|
XXX We can't make use of compiled libraries (Go \"packages\")."
|
|
(when install-source?
|
|
(if (string-null? import-path)
|
|
((display "WARNING: The Go import path is unset.\n")))
|
|
(let* ((out (assoc-ref outputs "out"))
|
|
(source (string-append (getenv "GOPATH") "/src/" import-path))
|
|
(dest (string-append out "/src/" import-path)))
|
|
(mkdir-p dest)
|
|
(copy-recursively source dest #:keep-mtime? #t)))
|
|
#t)
|
|
|
|
(define* (install-license-files #:key unpack-path
|
|
import-path
|
|
#:allow-other-keys
|
|
#:rest args)
|
|
"Install license files matching LICENSE-FILE-REGEXP to 'share/doc'. Adjust
|
|
the standard install-license-files phase to first enter the correct directory."
|
|
(with-directory-excursion (string-append "src/" (if (string-null? unpack-path)
|
|
import-path
|
|
unpack-path))
|
|
(apply (assoc-ref gnu:%standard-phases 'install-license-files) args)))
|
|
|
|
(define* (remove-store-reference file file-name
|
|
#:optional (store (%store-directory)))
|
|
"Remove from FILE occurrences of FILE-NAME in STORE; return #t when FILE-NAME
|
|
is encountered in FILE, #f otherwise. This implementation reads FILE one byte at
|
|
a time, which is slow. Instead, we should use the Boyer-Moore string search
|
|
algorithm; there is an example in (guix build grafts)."
|
|
(define pattern
|
|
(string-take file-name
|
|
(+ 34 (string-length (%store-directory)))))
|
|
|
|
(with-fluids ((%default-port-encoding #f))
|
|
(with-atomic-file-replacement file
|
|
(lambda (in out)
|
|
;; We cannot use `regexp-exec' here because it cannot deal with
|
|
;; strings containing NUL characters.
|
|
(format #t "removing references to `~a' from `~a'...~%" file-name file)
|
|
(setvbuf in 'block 65536)
|
|
(setvbuf out 'block 65536)
|
|
(fold-port-matches (lambda (match result)
|
|
(put-bytevector out (string->utf8 store))
|
|
(put-u8 out (char->integer #\/))
|
|
(put-bytevector out
|
|
(string->utf8
|
|
"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-"))
|
|
#t)
|
|
#f
|
|
pattern
|
|
in
|
|
(lambda (char result)
|
|
(put-u8 out (char->integer char))
|
|
result))))))
|
|
|
|
(define* (remove-go-references #:key allow-go-reference?
|
|
inputs outputs #:allow-other-keys)
|
|
"Remove any references to the Go compiler from the compiled Go executable
|
|
files in OUTPUTS."
|
|
;; We remove this spurious reference to save bandwidth when installing Go
|
|
;; executables. It would be better to not embed the reference in the first
|
|
;; place, but I'm not sure how to do that. The subject was discussed at:
|
|
;; <https://lists.gnu.org/archive/html/guix-devel/2017-10/msg00207.html>
|
|
(if allow-go-reference?
|
|
#t
|
|
(let ((go (assoc-ref inputs "go"))
|
|
(bin "/bin"))
|
|
(for-each (lambda (output)
|
|
(when (file-exists? (string-append (cdr output)
|
|
bin))
|
|
(for-each (lambda (file)
|
|
(remove-store-reference file go))
|
|
(find-files (string-append (cdr output) bin)))))
|
|
outputs)
|
|
#t)))
|
|
|
|
(define %standard-phases
|
|
(modify-phases gnu:%standard-phases
|
|
(delete 'bootstrap)
|
|
(delete 'configure)
|
|
(delete 'patch-generated-file-shebangs)
|
|
(add-before 'unpack 'setup-go-environment setup-go-environment)
|
|
(replace 'unpack unpack)
|
|
(replace 'build build)
|
|
(replace 'check check)
|
|
(replace 'install install)
|
|
(replace 'install-license-files install-license-files)
|
|
(add-after 'install 'remove-go-references remove-go-references)))
|
|
|
|
(define* (go-build #:key inputs (phases %standard-phases)
|
|
#:allow-other-keys #:rest args)
|
|
"Build the given Go package, applying all of PHASES in order."
|
|
(apply gnu:gnu-build #:inputs inputs #:phases phases args))
|