From 2d65e5b6a624e9642c4d765a861cf291095adc72 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 12 Apr 2024 06:43:57 +0200 Subject: [PATCH 01/13] ci: rename "runs_on_pool" to "distro" The "runs_on_pool" environment variable is used by our CI scripts to distinguish the different kinds of operating systems. It is quite specific to GitHub Actions though and not really a descriptive name. Rename the variable to "distro" to clarify its intent. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- .github/workflows/main.yml | 2 +- ci/install-dependencies.sh | 2 +- ci/lib.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3428773b09..684ef5c00d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -303,7 +303,7 @@ jobs: CC: ${{matrix.vector.cc}} CC_PACKAGE: ${{matrix.vector.cc_package}} jobname: ${{matrix.vector.jobname}} - runs_on_pool: ${{matrix.vector.pool}} + distro: ${{matrix.vector.pool}} runs-on: ${{matrix.vector.pool}} steps: - uses: actions/checkout@v4 diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index b4e22de3cb..7d247b5ef4 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -11,7 +11,7 @@ UBUNTU_COMMON_PKGS="make libssl-dev libcurl4-openssl-dev libexpat-dev tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl libemail-valid-perl libio-socket-ssl-perl libnet-smtp-ssl-perl" -case "$runs_on_pool" in +case "$distro" in ubuntu-*) sudo apt-get -q update sudo apt-get -q -y install language-pack-is libsvn-perl apache2 \ diff --git a/ci/lib.sh b/ci/lib.sh index 0a73fc7bd1..d882250db5 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -279,7 +279,7 @@ then cache_dir="$HOME/none" - runs_on_pool=$(echo "$CI_JOB_IMAGE" | tr : -) + distro=$(echo "$CI_JOB_IMAGE" | tr : -) JOBS=$(nproc) else echo "Could not identify CI type" >&2 @@ -318,7 +318,7 @@ export DEFAULT_TEST_TARGET=prove export GIT_TEST_CLONE_2GB=true export SKIP_DASHED_BUILT_INS=YesPlease -case "$runs_on_pool" in +case "$distro" in ubuntu-*) if test "$jobname" = "linux-gcc-default" then From ab2b3aadf322fccffeaabe7470e1b5905537a888 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 12 Apr 2024 06:44:02 +0200 Subject: [PATCH 02/13] ci: expose distro name in dockerized GitHub jobs Expose a distro name in dockerized jobs. This will be used in a subsequent commit where we merge the installation scripts for dockerized and non-dockerized jobs. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- .github/workflows/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 684ef5c00d..71cd4e5486 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -342,12 +342,16 @@ jobs: vector: - jobname: linux-musl image: alpine + distro: alpine-latest - jobname: linux32 image: daald/ubuntu32:xenial + distro: ubuntu32-16.04 - jobname: pedantic image: fedora + distro: fedora-latest env: jobname: ${{matrix.vector.jobname}} + distro: ${{matrix.vector.distro}} runs-on: ubuntu-latest container: ${{matrix.vector.image}} steps: From 11d3f1aa5fce1838483a48e9fbabbd762e0dda97 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 12 Apr 2024 06:44:07 +0200 Subject: [PATCH 03/13] ci: skip sudo when we are already root Our "install-dependencies.sh" script is executed by non-dockerized jobs to install dependencies. These jobs don't run with "root" permissions, but with a separate user. Consequently, we need to use sudo(8) there to elevate permissions when installing packages. We're about to merge "install-docker-dependencies.sh" into that script though, and our Docker containers do run as "root". Using sudo(8) is thus unnecessary there, even though it would be harmless. On some images like Alpine Linux though there is no sudo(8) available by default, which would consequently break the build. Adapt the script to make "sudo" a no-op when running as "root" user. This allows us to easily reuse the script for our dockerized jobs. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- ci/install-dependencies.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index 7d247b5ef4..7dfd3e50ed 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -11,6 +11,17 @@ UBUNTU_COMMON_PKGS="make libssl-dev libcurl4-openssl-dev libexpat-dev tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl libemail-valid-perl libio-socket-ssl-perl libnet-smtp-ssl-perl" +# Make sudo a no-op and execute the command directly when running as root. +# While using sudo would be fine on most platforms when we are root already, +# some platforms like e.g. Alpine Linux do not have sudo available by default +# and would thus break. +if test "$(id -u)" -eq 0 +then + sudo () { + "$@" + } +fi + case "$distro" in ubuntu-*) sudo apt-get -q update From 21bcb4a6023c70b1359ba138dedcf37939be5eaa Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 12 Apr 2024 06:44:12 +0200 Subject: [PATCH 04/13] ci: drop duplicate package installation for "linux-gcc-default" The "linux-gcc-default" job installs common Ubuntu packages. This is already done in the distro-specific switch, so we basically duplicate the effort here. Drop the duplicate package installations and inline the variable that contains those common packages. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- ci/install-dependencies.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index 7dfd3e50ed..fad53aac96 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -7,9 +7,6 @@ P4WHENCE=https://cdist2.perforce.com/perforce/r21.2 LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION -UBUNTU_COMMON_PKGS="make libssl-dev libcurl4-openssl-dev libexpat-dev - tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl - libemail-valid-perl libio-socket-ssl-perl libnet-smtp-ssl-perl" # Make sudo a no-op and execute the command directly when running as root. # While using sudo would be fine on most platforms when we are root already, @@ -25,8 +22,13 @@ fi case "$distro" in ubuntu-*) sudo apt-get -q update - sudo apt-get -q -y install language-pack-is libsvn-perl apache2 \ - $UBUNTU_COMMON_PKGS $CC_PACKAGE $PYTHON_PACKAGE + sudo apt-get -q -y install \ + language-pack-is libsvn-perl apache2 \ + make libssl-dev libcurl4-openssl-dev libexpat-dev \ + tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl \ + libemail-valid-perl libio-socket-ssl-perl libnet-smtp-ssl-perl \ + $CC_PACKAGE $PYTHON_PACKAGE + mkdir --parents "$P4_PATH" pushd "$P4_PATH" wget --quiet "$P4WHENCE/bin.linux26x86_64/p4d" @@ -34,6 +36,7 @@ ubuntu-*) chmod u+x p4d chmod u+x p4 popd + mkdir --parents "$GIT_LFS_PATH" pushd "$GIT_LFS_PATH" wget --quiet "$LFSWHENCE/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" @@ -83,10 +86,6 @@ Documentation) test -n "$ALREADY_HAVE_ASCIIDOCTOR" || sudo gem install --version 1.5.8 asciidoctor ;; -linux-gcc-default) - sudo apt-get -q update - sudo apt-get -q -y install $UBUNTU_COMMON_PKGS - ;; esac if type p4d >/dev/null 2>&1 && type p4 >/dev/null 2>&1 From 40c60f4c124d1ae59c0f05dcd37dbe0c63d64ab7 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 12 Apr 2024 06:44:17 +0200 Subject: [PATCH 05/13] ci: convert "install-dependencies.sh" to use "/bin/sh" We're about to merge the "install-docker-dependencies.sh" script into "install-dependencies.sh". This will also move our Alpine-based jobs over to use the latter script. This script uses the Bash shell though, which is not available by default on Alpine Linux. Refactor "install-dependencies.sh" to use "/bin/sh" instead of Bash. This requires us to get rid of the pushd/popd invocations, which are replaced by some more elaborate commands that download or extract executables right to where they are needed. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- ci/install-dependencies.sh | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index fad53aac96..2d6af876d6 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/sh # # Install dependencies required to build and test Git on Linux and macOS # @@ -30,19 +30,15 @@ ubuntu-*) $CC_PACKAGE $PYTHON_PACKAGE mkdir --parents "$P4_PATH" - pushd "$P4_PATH" - wget --quiet "$P4WHENCE/bin.linux26x86_64/p4d" - wget --quiet "$P4WHENCE/bin.linux26x86_64/p4" - chmod u+x p4d - chmod u+x p4 - popd + wget --quiet --directory-prefix="$P4_PATH" \ + "$P4WHENCE/bin.linux26x86_64/p4d" "$P4WHENCE/bin.linux26x86_64/p4" + chmod u+x "$P4_PATH/p4d" "$P4_PATH/p4" mkdir --parents "$GIT_LFS_PATH" - pushd "$GIT_LFS_PATH" - wget --quiet "$LFSWHENCE/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" - tar --extract --gunzip --file "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" - cp git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs . - popd + wget --quiet "$LFSWHENCE/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" + tar -xzf "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" \ + -C "$GIT_LFS_PATH" --strip-components=1 "git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs" + rm "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" ;; macos-*) export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 @@ -53,11 +49,10 @@ macos-*) brew link --force gettext mkdir -p "$P4_PATH" - pushd "$P4_PATH" - wget -q "$P4WHENCE/bin.macosx1015x86_64/helix-core-server.tgz" && - tar -xf helix-core-server.tgz && - sudo xattr -d com.apple.quarantine p4 p4d 2>/dev/null || true - popd + wget -q "$P4WHENCE/bin.macosx1015x86_64/helix-core-server.tgz" && + tar -xf helix-core-server.tgz -C "$P4_PATH" p4 p4d && + sudo xattr -d com.apple.quarantine "$P4_PATH/p4" "$P4_PATH/p4d" 2>/dev/null || true + rm helix-core-server.tgz if test -n "$CC_PACKAGE" then From d1ef3d3b1d562e3362de0886b18c8cc2df85c4b7 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 12 Apr 2024 06:44:22 +0200 Subject: [PATCH 06/13] ci: merge custom PATH directories We're downloading various executables required by our tests. Each of these executables goes into its own directory, which is then appended to the PATH variable. Consequently, whenever we add a new dependency and thus a new directory, we would have to adapt to this change in several places. Refactor this to instead put all binaries into a single directory. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- ci/install-dependencies.sh | 15 +++++++-------- ci/lib.sh | 10 +++------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index 2d6af876d6..bafe37f2d1 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -29,15 +29,14 @@ ubuntu-*) libemail-valid-perl libio-socket-ssl-perl libnet-smtp-ssl-perl \ $CC_PACKAGE $PYTHON_PACKAGE - mkdir --parents "$P4_PATH" - wget --quiet --directory-prefix="$P4_PATH" \ + mkdir --parents "$CUSTOM_PATH" + wget --quiet --directory-prefix="$CUSTOM_PATH" \ "$P4WHENCE/bin.linux26x86_64/p4d" "$P4WHENCE/bin.linux26x86_64/p4" - chmod u+x "$P4_PATH/p4d" "$P4_PATH/p4" + chmod u+x "$CUSTOM_PATH/p4d" "$CUSTOM_PATH/p4" - mkdir --parents "$GIT_LFS_PATH" wget --quiet "$LFSWHENCE/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" tar -xzf "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" \ - -C "$GIT_LFS_PATH" --strip-components=1 "git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs" + -C "$CUSTOM_PATH" --strip-components=1 "git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs" rm "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" ;; macos-*) @@ -48,10 +47,10 @@ macos-*) brew install $BREW_INSTALL_PACKAGES brew link --force gettext - mkdir -p "$P4_PATH" + mkdir -p "$CUSTOM_PATH" wget -q "$P4WHENCE/bin.macosx1015x86_64/helix-core-server.tgz" && - tar -xf helix-core-server.tgz -C "$P4_PATH" p4 p4d && - sudo xattr -d com.apple.quarantine "$P4_PATH/p4" "$P4_PATH/p4d" 2>/dev/null || true + tar -xf helix-core-server.tgz -C "$CUSTOM_PATH" p4 p4d && + sudo xattr -d com.apple.quarantine "$CUSTOM_PATH/p4" "$CUSTOM_PATH/p4d" 2>/dev/null || true rm helix-core-server.tgz if test -n "$CC_PACKAGE" diff --git a/ci/lib.sh b/ci/lib.sh index d882250db5..4cce854bad 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -340,10 +340,6 @@ ubuntu-*) # image. # Keep that in mind when you encounter a broken OS X build! export LINUX_GIT_LFS_VERSION="1.5.2" - - P4_PATH="$HOME/custom/p4" - GIT_LFS_PATH="$HOME/custom/git-lfs" - export PATH="$GIT_LFS_PATH:$P4_PATH:$PATH" ;; macos-*) MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)" @@ -351,12 +347,12 @@ macos-*) then MAKEFLAGS="$MAKEFLAGS APPLE_COMMON_CRYPTO_SHA1=Yes" fi - - P4_PATH="$HOME/custom/p4" - export PATH="$P4_PATH:$PATH" ;; esac +CUSTOM_PATH="$HOME/path" +export PATH="$CUSTOM_PATH:$PATH" + case "$jobname" in linux32) CC=gcc From 2c5c7639e59a73f1428fa6fb5dff647234046f59 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 12 Apr 2024 06:44:27 +0200 Subject: [PATCH 07/13] ci: fix setup of custom path for GitLab CI Part of "install-dependencies.sh" is to install some binaries required for tests into a custom directory that gets added to the PATH. This directory is located at "$HOME/path" and thus depends on the current user that the script executes as. This creates problems for GitLab CI, which installs dependencies as the root user, but runs tests as a separate, unprivileged user. As their respective home directories are different, we will end up using two different custom path directories. Consequently, the unprivileged user will not be able to find the binaries that were set up as root user. Fix this issue by allowing CI to override the custom path, which allows GitLab to set up a constant value that isn't derived from "$HOME". Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- .gitlab-ci.yml | 2 ++ ci/lib.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c0fa2fe90b..3a0ef4d4d4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,8 @@ workflow: test:linux: image: $image + variables: + CUSTOM_PATH: "/custom" before_script: - ./ci/install-docker-dependencies.sh script: diff --git a/ci/lib.sh b/ci/lib.sh index 4cce854bad..473a2d0348 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -350,7 +350,7 @@ macos-*) ;; esac -CUSTOM_PATH="$HOME/path" +CUSTOM_PATH="${CUSTOM_PATH:-$HOME/path}" export PATH="$CUSTOM_PATH:$PATH" case "$jobname" in From 9cdeb34b962d5c1633ac5d23ebba4e24a816242a Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 12 Apr 2024 06:44:32 +0200 Subject: [PATCH 08/13] ci: merge scripts which install dependencies We have two different scripts which install dependencies, one for dockerized jobs and one for non-dockerized ones. Naturally, these scripts have quite some duplication. Furthermore, either of these scripts is missing some test dependencies that the respective other script has, thus reducing test coverage. Merge those two scripts such that there is a single source of truth for test dependencies, only. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- .github/workflows/main.yml | 2 +- .gitlab-ci.yml | 4 +-- ci/install-dependencies.sh | 32 ++++++++++++++++++--- ci/install-docker-dependencies.sh | 46 ------------------------------- 4 files changed, 31 insertions(+), 53 deletions(-) delete mode 100755 ci/install-docker-dependencies.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 71cd4e5486..5838986895 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -359,7 +359,7 @@ jobs: if: matrix.vector.jobname != 'linux32' - uses: actions/checkout@v1 # cannot be upgraded because Node.js Actions aren't supported in this container if: matrix.vector.jobname == 'linux32' - - run: ci/install-docker-dependencies.sh + - run: ci/install-dependencies.sh - run: ci/run-build-and-tests.sh - name: print test failures if: failure() && env.FAILED_TEST_ARTIFACTS != '' diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3a0ef4d4d4..ba65f50aac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,7 +12,7 @@ test:linux: variables: CUSTOM_PATH: "/custom" before_script: - - ./ci/install-docker-dependencies.sh + - ./ci/install-dependencies.sh script: - useradd builder --create-home - chown -R builder "${CI_PROJECT_DIR}" @@ -100,7 +100,7 @@ static-analysis: variables: jobname: StaticAnalysis before_script: - - ./ci/install-docker-dependencies.sh + - ./ci/install-dependencies.sh script: - ./ci/run-static-analysis.sh - ./ci/check-directional-formatting.bash diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index bafe37f2d1..e673797115 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -5,6 +5,8 @@ . ${0%/*}/lib.sh +begin_group "Install dependencies" + P4WHENCE=https://cdist2.perforce.com/perforce/r21.2 LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION @@ -20,14 +22,27 @@ then fi case "$distro" in +alpine-*) + apk add --update shadow sudo build-base curl-dev openssl-dev expat-dev gettext \ + pcre2-dev python3 musl-libintl perl-utils ncurses \ + apache2 apache2-http2 apache2-proxy apache2-ssl apache2-webdav apr-util-dbd_sqlite3 \ + bash cvs gnupg perl-cgi perl-dbd-sqlite >/dev/null + ;; +fedora-*) + dnf -yq update >/dev/null && + dnf -yq install make gcc findutils diffutils perl python3 gettext zlib-devel expat-devel openssl-devel curl-devel pcre2-devel >/dev/null + ;; ubuntu-*) + # Required so that apt doesn't wait for user input on certain packages. + export DEBIAN_FRONTEND=noninteractive + sudo apt-get -q update sudo apt-get -q -y install \ - language-pack-is libsvn-perl apache2 \ - make libssl-dev libcurl4-openssl-dev libexpat-dev \ + language-pack-is libsvn-perl apache2 cvs cvsps git gnupg subversion \ + make libssl-dev libcurl4-openssl-dev libexpat-dev wget sudo \ tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl \ - libemail-valid-perl libio-socket-ssl-perl libnet-smtp-ssl-perl \ - $CC_PACKAGE $PYTHON_PACKAGE + libemail-valid-perl libio-socket-ssl-perl libnet-smtp-ssl-perl libdbd-sqlite3-perl libcgi-pm-perl \ + ${CC_PACKAGE:-${CC:-gcc}} $PYTHON_PACKAGE mkdir --parents "$CUSTOM_PATH" wget --quiet --directory-prefix="$CUSTOM_PATH" \ @@ -39,6 +54,13 @@ ubuntu-*) -C "$CUSTOM_PATH" --strip-components=1 "git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs" rm "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" ;; +ubuntu32-*) + sudo linux32 --32bit i386 sh -c ' + apt update >/dev/null && + apt install -y build-essential libcurl4-openssl-dev \ + libssl-dev libexpat-dev gettext python >/dev/null + ' + ;; macos-*) export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 # Uncomment this if you want to run perf tests: @@ -98,3 +120,5 @@ then else echo >&2 "WARNING: git-lfs wasn't installed, see above for clues why" fi + +end_group "Install dependencies" diff --git a/ci/install-docker-dependencies.sh b/ci/install-docker-dependencies.sh deleted file mode 100755 index eb2c9e1eca..0000000000 --- a/ci/install-docker-dependencies.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh -# -# Install dependencies required to build and test Git inside container -# - -. ${0%/*}/lib.sh - -begin_group "Install dependencies" - -case "$jobname" in -linux32) - linux32 --32bit i386 sh -c ' - apt update >/dev/null && - apt install -y build-essential libcurl4-openssl-dev \ - libssl-dev libexpat-dev gettext python >/dev/null - ' - ;; -linux-musl) - apk add --update shadow sudo build-base curl-dev openssl-dev expat-dev gettext \ - pcre2-dev python3 musl-libintl perl-utils ncurses \ - apache2 apache2-http2 apache2-proxy apache2-ssl apache2-webdav apr-util-dbd_sqlite3 \ - bash cvs gnupg perl-cgi perl-dbd-sqlite >/dev/null - ;; -linux-*|StaticAnalysis) - # Required so that apt doesn't wait for user input on certain packages. - export DEBIAN_FRONTEND=noninteractive - - apt update -q && - apt install -q -y sudo git make language-pack-is libsvn-perl apache2 libssl-dev \ - libcurl4-openssl-dev libexpat-dev tcl tk gettext zlib1g-dev \ - perl-modules liberror-perl libauthen-sasl-perl libemail-valid-perl \ - libdbd-sqlite3-perl libio-socket-ssl-perl libnet-smtp-ssl-perl ${CC_PACKAGE:-${CC:-gcc}} \ - apache2 cvs cvsps gnupg libcgi-pm-perl subversion - - if test "$jobname" = StaticAnalysis - then - apt install -q -y coccinelle - fi - ;; -pedantic) - dnf -yq update >/dev/null && - dnf -yq install make gcc findutils diffutils perl python3 gettext zlib-devel expat-devel openssl-devel curl-devel pcre2-devel >/dev/null - ;; -esac - -end_group "Install dependencies" From ca44ef3165324ff76aa27261141f573846f9bfca Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 12 Apr 2024 06:44:37 +0200 Subject: [PATCH 09/13] ci: make Perforce binaries executable for all users The Perforce binaries are only made executable for the current user. On GitLab CI though we execute tests as a different user than "root", and thus these binaries may not be executable by that test user at all. This has gone unnoticed so far because those binaries are optional -- in case they don't exist we simply skip over tests requiring them. Fix the setup so that we set the executable bits for all users. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- ci/install-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index e673797115..0b9bb686d8 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -47,7 +47,7 @@ ubuntu-*) mkdir --parents "$CUSTOM_PATH" wget --quiet --directory-prefix="$CUSTOM_PATH" \ "$P4WHENCE/bin.linux26x86_64/p4d" "$P4WHENCE/bin.linux26x86_64/p4" - chmod u+x "$CUSTOM_PATH/p4d" "$CUSTOM_PATH/p4" + chmod a+x "$CUSTOM_PATH/p4d" "$CUSTOM_PATH/p4" wget --quiet "$LFSWHENCE/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" tar -xzf "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" \ From 04ba2c7eb38fbb0f1a440971953557b9eb3b789b Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 12 Apr 2024 06:44:43 +0200 Subject: [PATCH 10/13] ci: install JGit dependency We have some tests in t5310 that use JGit to verify that bitmaps can be read both by Git and by JGit. We do not execute these tests in our CI jobs though because we don't make JGit available there. Consequently, the tests basically bitrot because almost nobody is ever going to have JGit in their path. Install JGit to plug this test gap. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- ci/install-dependencies.sh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index 0b9bb686d8..c196e56762 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -9,6 +9,7 @@ begin_group "Install dependencies" P4WHENCE=https://cdist2.perforce.com/perforce/r21.2 LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION +JGITWHENCE=https://repo.eclipse.org/content/groups/releases//org/eclipse/jgit/org.eclipse.jgit.pgm/6.8.0.202311291450-r/org.eclipse.jgit.pgm-6.8.0.202311291450-r.sh # Make sudo a no-op and execute the command directly when running as root. # While using sudo would be fine on most platforms when we are root already, @@ -39,7 +40,7 @@ ubuntu-*) sudo apt-get -q update sudo apt-get -q -y install \ language-pack-is libsvn-perl apache2 cvs cvsps git gnupg subversion \ - make libssl-dev libcurl4-openssl-dev libexpat-dev wget sudo \ + make libssl-dev libcurl4-openssl-dev libexpat-dev wget sudo default-jre \ tcl tk gettext zlib1g-dev perl-modules liberror-perl libauthen-sasl-perl \ libemail-valid-perl libio-socket-ssl-perl libnet-smtp-ssl-perl libdbd-sqlite3-perl libcgi-pm-perl \ ${CC_PACKAGE:-${CC:-gcc}} $PYTHON_PACKAGE @@ -53,6 +54,9 @@ ubuntu-*) tar -xzf "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" \ -C "$CUSTOM_PATH" --strip-components=1 "git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs" rm "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" + + wget --quiet "$JGITWHENCE" --output-document="$CUSTOM_PATH/jgit" + chmod a+x "$CUSTOM_PATH/jgit" ;; ubuntu32-*) sudo linux32 --32bit i386 sh -c ' @@ -113,6 +117,7 @@ then else echo >&2 "WARNING: perforce wasn't installed, see above for clues why" fi + if type git-lfs >/dev/null 2>&1 then echo "$(tput setaf 6)Git-LFS Version$(tput sgr0)" @@ -121,4 +126,12 @@ else echo >&2 "WARNING: git-lfs wasn't installed, see above for clues why" fi +if type jgit >/dev/null 2>&1 +then + echo "$(tput setaf 6)JGit Version$(tput sgr0)" + jgit version +else + echo >&2 "WARNING: JGit wasn't installed, see above for clues why" +fi + end_group "Install dependencies" From ca13c3e94a737b3dc8624e433ed87fead3a8b974 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 12 Apr 2024 06:44:48 +0200 Subject: [PATCH 11/13] t06xx: always execute backend-specific tests The tests in t06xx exercise specific ref formats. Next to probing some basic functionality, these tests also exercise other low-level details specific to the format. Those tests are only executed though in case `GIT_TEST_DEFAULT_REF_FORMAT` is set to the ref format of the respective backend-under-test. Ideally, we would run the full test matrix for ref formats such that our complete test suite is executed with every supported format on every supported platform. This is quite an expensive undertaking though, and thus we only execute e.g. the "reftable" tests on macOS and Linux. As a result, we basically have no test coverage for the "reftable" format at all on other platforms like Windows. Adapt these tests so that they override `GIT_TEST_DEFAULT_REF_FORMAT`, which means that they'll always execute. This increases test coverage on platforms that don't run the full test matrix, which at least gives us some basic test coverage on those platforms for the "reftable" format. This of course comes at the cost of running those tests multiple times on platforms where we do run the full test matrix. But arguably, this is a good thing because it will also cause us to e.g. run those tests with the address sanitizer and other non-standard parameters. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- t/t0600-reffiles-backend.sh | 8 ++------ t/t0601-reffiles-pack-refs.sh | 9 +++------ t/t0610-reftable-basics.sh | 9 +++------ 3 files changed, 8 insertions(+), 18 deletions(-) diff --git a/t/t0600-reffiles-backend.sh b/t/t0600-reffiles-backend.sh index 64214340e7..a390cffc80 100755 --- a/t/t0600-reffiles-backend.sh +++ b/t/t0600-reffiles-backend.sh @@ -4,16 +4,12 @@ test_description='Test reffiles backend' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +GIT_TEST_DEFAULT_REF_FORMAT=files +export GIT_TEST_DEFAULT_REF_FORMAT TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh -if ! test_have_prereq REFFILES -then - skip_all='skipping reffiles specific tests' - test_done -fi - test_expect_success 'setup' ' git commit --allow-empty -m Initial && C=$(git rev-parse HEAD) && diff --git a/t/t0601-reffiles-pack-refs.sh b/t/t0601-reffiles-pack-refs.sh index c309d2bae8..157f79fe52 100755 --- a/t/t0601-reffiles-pack-refs.sh +++ b/t/t0601-reffiles-pack-refs.sh @@ -9,18 +9,15 @@ test_description='git pack-refs should not change the branch semantic This test runs git pack-refs and git show-ref and checks that the branch semantic is still the same. ' + GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +GIT_TEST_DEFAULT_REF_FORMAT=files +export GIT_TEST_DEFAULT_REF_FORMAT TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh -if ! test_have_prereq REFFILES -then - skip_all='skipping reffiles specific tests' - test_done -fi - test_expect_success 'enable reflogs' ' git config core.logallrefupdates true ' diff --git a/t/t0610-reftable-basics.sh b/t/t0610-reftable-basics.sh index 686781192e..aa9282007c 100755 --- a/t/t0610-reftable-basics.sh +++ b/t/t0610-reftable-basics.sh @@ -4,17 +4,14 @@ # test_description='reftable basics' + GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +GIT_TEST_DEFAULT_REF_FORMAT=reftable +export GIT_TEST_DEFAULT_REF_FORMAT . ./test-lib.sh -if ! test_have_prereq REFTABLE -then - skip_all='skipping reftable tests; set GIT_TEST_DEFAULT_REF_FORMAT=reftable' - test_done -fi - INVALID_OID=$(test_oid 001) test_expect_success 'init: creates basic reftable structures' ' From db1d63bf57b49d71b7327f36740430525d7049b0 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 12 Apr 2024 06:44:53 +0200 Subject: [PATCH 12/13] t0610: fix non-portable variable assignment Older versions of the Dash shell fail to parse `local var=val` assignments in some cases when `val` is unquoted. Such failures can be observed e.g. with Ubuntu 20.04 and older, which has a Dash version that still has this bug. Such an assignment has been introduced in t0610. The issue wasn't detected for a while because this test used to only run when the GIT_TEST_DEFAULT_REF_FORMAT environment variable was set to "reftable". We have dropped that requirement now though, meaning that it runs unconditionally, including on jobs which use such older versions of Ubuntu. We have worked around such issues in the past, e.g. in ebee5580ca (parallel-checkout: avoid dash local bug in tests, 2021-06-06), by quoting the `val` side. Apply the same fix to t0610. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- t/t0610-reftable-basics.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/t/t0610-reftable-basics.sh b/t/t0610-reftable-basics.sh index aa9282007c..dfa7e274ea 100755 --- a/t/t0610-reftable-basics.sh +++ b/t/t0610-reftable-basics.sh @@ -78,9 +78,9 @@ test_expect_success 'init: reinitializing reftable with files backend fails' ' ' test_expect_perms () { - local perms="$1" - local file="$2" - local actual=$(ls -l "$file") && + local perms="$1" && + local file="$2" && + local actual="$(ls -l "$file")" && case "$actual" in $perms*) From 70b81fbf3c641435c5583e53ff78caa0aaef80ca Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Fri, 12 Apr 2024 06:44:58 +0200 Subject: [PATCH 13/13] t0612: add tests to exercise Git/JGit reftable compatibility While the reftable format is a recent introduction in Git, JGit already knows to read and write reftables since 2017. Given the complexity of the format there is a very real risk of incompatibilities between those two implementations, which is something that we really want to avoid. Add some basic tests that verify that reftables written by Git and JGit can be read by the respective other implementation. For now this test suite is rather small, only covering basic functionality. But it serves as a good starting point and can be extended over time. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- t/t0612-reftable-jgit-compatibility.sh | 132 +++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100755 t/t0612-reftable-jgit-compatibility.sh diff --git a/t/t0612-reftable-jgit-compatibility.sh b/t/t0612-reftable-jgit-compatibility.sh new file mode 100755 index 0000000000..d0d7e80b49 --- /dev/null +++ b/t/t0612-reftable-jgit-compatibility.sh @@ -0,0 +1,132 @@ +#!/bin/sh + +test_description='reftables are compatible with JGit' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME +GIT_TEST_DEFAULT_REF_FORMAT=reftable +export GIT_TEST_DEFAULT_REF_FORMAT + +# JGit does not support the 'link' DIRC extension. +GIT_TEST_SPLIT_INDEX=0 +export GIT_TEST_SPLIT_INDEX + +. ./test-lib.sh + +if ! test_have_prereq JGIT +then + skip_all='skipping reftable JGit tests; JGit is not present in PATH' + test_done +fi + +if ! test_have_prereq SHA1 +then + skip_all='skipping reftable JGit tests; JGit does not support SHA256 reftables' + test_done +fi + +test_commit_jgit () { + touch "$1" && + jgit add "$1" && + jgit commit -m "$1" +} + +test_same_refs () { + git show-ref --head >cgit.actual && + jgit show-ref >jgit-tabs.actual && + tr "\t" " " jgit.actual && + test_cmp cgit.actual jgit.actual +} + +test_same_ref () { + git rev-parse "$1" >cgit.actual && + jgit rev-parse "$1" >jgit.actual && + test_cmp cgit.actual jgit.actual +} + +test_same_reflog () { + git reflog "$*" >cgit.actual && + jgit reflog "$*" >jgit-newline.actual && + sed '/^$/d' jgit.actual && + test_cmp cgit.actual jgit.actual +} + +test_expect_success 'CGit repository can be read by JGit' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit A && + test_same_refs && + test_same_ref HEAD && + test_same_reflog HEAD + ) +' + +test_expect_success 'JGit repository can be read by CGit' ' + test_when_finished "rm -rf repo" && + jgit init repo && + ( + cd repo && + + touch file && + jgit add file && + jgit commit -m "initial commit" && + + # Note that we must convert the ref storage after we have + # written the default branch. Otherwise JGit will end up with + # no HEAD at all. + jgit convert-ref-storage --format=reftable && + + test_same_refs && + test_same_ref HEAD && + # Interestingly, JGit cannot read its own reflog here. CGit can + # though. + printf "%s HEAD@{0}: commit (initial): initial commit" "$(git rev-parse --short HEAD)" >expect && + git reflog HEAD >actual && + test_cmp expect actual + ) +' + +test_expect_success 'mixed writes from JGit and CGit' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + + test_commit A && + test_commit_jgit B && + test_commit C && + test_commit_jgit D && + + test_same_refs && + test_same_ref HEAD && + test_same_reflog HEAD + ) +' + +test_expect_success 'JGit can read multi-level index' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + + test_commit A && + awk " + BEGIN { + print \"start\"; + for (i = 0; i < 10000; i++) + printf \"create refs/heads/branch-%d HEAD\n\", i; + print \"commit\"; + } + " >input && + git update-ref --stdin