Sync with 2.20.5

* maint-2.20:
  Git 2.20.5
  Git 2.19.6
  Git 2.18.5
  Git 2.17.6
  unpack_trees(): start with a fresh lstat cache
  run-command: invalidate lstat cache after a command finished
  checkout: fix bug that makes checkout follow symlinks in leading path

Documentation/RelNotes/2.17.6.txt

@@ -0,0 +1,16 @@
+Git v2.17.6 Release Notes
+=========================
+
+This release addresses the security issues CVE-2021-21300.
+
+Fixes since v2.17.5
+-------------------
+
+ * CVE-2021-21300:
+   On case-insensitive file systems with support for symbolic links,
+   if Git is configured globally to apply delay-capable clean/smudge
+   filters (such as Git LFS), Git could be fooled into running
+   remote code during a clone.
+
+Credit for finding and fixing this vulnerability goes to Matheus
+Tavares, helped by Johannes Schindelin.

Documentation/RelNotes/2.18.5.txt

@@ -0,0 +1,6 @@
+Git v2.18.5 Release Notes
+=========================
+
+This release merges up the fixes that appear in v2.17.6 to address
+the security issue CVE-2021-21300; see the release notes for that
+version for details.

Documentation/RelNotes/2.19.6.txt

@@ -0,0 +1,6 @@
+Git v2.19.6 Release Notes
+=========================
+
+This release merges up the fixes that appear in v2.17.6 and
+v2.18.5 to address the security issue CVE-2021-21300; see the
+release notes for these versions for details.

Documentation/RelNotes/2.20.5.txt

@@ -0,0 +1,6 @@
+Git v2.20.5 Release Notes
+=========================
+
+This release merges up the fixes that appear in v2.17.6, v2.18.5
+and v2.19.6 to address the security issue CVE-2021-21300; see
+the release notes for these versions for details.

cache.h

@@ -1586,6 +1586,7 @@ extern int has_symlink_leading_path(const char *name, int len);
 extern int threaded_has_symlink_leading_path(struct cache_def *, const char *, int);
 extern int check_leading_path(const char *name, int len);
 extern int has_dirs_only_path(const char *name, int len, int prefix_len);
+extern void invalidate_lstat_cache(void);
 extern void schedule_dir_for_removal(const char *name, int len);
 extern void remove_scheduled_dirs(void);
 

compat/mingw.c

@@ -340,6 +340,8 @@ int mingw_rmdir(const char *pathname)
 	       ask_yes_no_if_possible("Deletion of directory '%s' failed. "
 			"Should I try again?", pathname))
 	       ret = _wrmdir(wpathname);
+	if (!ret)
+		invalidate_lstat_cache();
 	return ret;
 }
 

git-compat-util.h

@@ -354,6 +354,11 @@ static inline int noop_core_config(const char *var, const char *value, void *cb)
 #define platform_core_config noop_core_config
 #endif
 
+int lstat_cache_aware_rmdir(const char *path);
+#if !defined(__MINGW32__) && !defined(_MSC_VER)
+#define rmdir lstat_cache_aware_rmdir
+#endif
+
 #ifndef has_dos_drive_prefix
 static inline int git_has_dos_drive_prefix(const char *path)
 {

run-command.c

@@ -965,6 +965,7 @@ int finish_command(struct child_process *cmd)
 {
 	int ret = wait_or_whine(cmd->pid, cmd->argv[0], 0);
 	child_process_clear(cmd);
+	invalidate_lstat_cache();
 	return ret;
 }
 
@@ -1251,13 +1252,19 @@ int start_async(struct async *async)
 int finish_async(struct async *async)
 {
 #ifdef NO_PTHREADS
-	return wait_or_whine(async->pid, "child process", 0);
+	int ret = wait_or_whine(async->pid, "child process", 0);
+
+	invalidate_lstat_cache();
+
+	return ret;
 #else
 	void *ret = (void *)(intptr_t)(-1);
 
 	if (pthread_join(async->tid, &ret))
 		error("pthread_join failed");
+	invalidate_lstat_cache();
 	return (int)(intptr_t)ret;
+
 #endif
 }
 

symlinks.c

@@ -267,6 +267,13 @@ int has_dirs_only_path(const char *name, int len, int prefix_len)
  */
 static int threaded_has_dirs_only_path(struct cache_def *cache, const char *name, int len, int prefix_len)
 {
+	/*
+	 * Note: this function is used by the checkout machinery, which also
+	 * takes care to properly reset the cache when it performs an operation
+	 * that would leave the cache outdated. If this function starts caching
+	 * anything else besides FL_DIR, remember to also invalidate the cache
+	 * when creating or deleting paths that might be in the cache.
+	 */
 	return lstat_cache(cache, name, len,
 			   FL_DIR|FL_FULLPATH, prefix_len) &
 		FL_DIR;
@@ -321,3 +328,20 @@ void remove_scheduled_dirs(void)
 {
 	do_remove_scheduled_dirs(0);
 }
+
+void invalidate_lstat_cache(void)
+{
+	reset_lstat_cache(&default_cache);
+}
+
+#undef rmdir
+int lstat_cache_aware_rmdir(const char *path)
+{
+	/* Any change in this function must be made also in `mingw_rmdir()` */
+	int ret = rmdir(path);
+
+	if (!ret)
+		invalidate_lstat_cache();
+
+	return ret;
+}

t/t0021-conversion.sh

@@ -817,4 +817,85 @@ test_expect_success PERL 'invalid file in delayed checkout' '
 	grep "error: external filter .* signaled that .unfiltered. is now available although it has not been delayed earlier" git-stderr.log
 '
 
+for mode in 'case' 'utf-8'
+do
+	case "$mode" in
+	case)	dir='A' symlink='a' mode_prereq='CASE_INSENSITIVE_FS' ;;
+	utf-8)
+		dir=$(printf "\141\314\210") symlink=$(printf "\303\244")
+		mode_prereq='UTF8_NFD_TO_NFC' ;;
+	esac
+
+	test_expect_success PERL,SYMLINKS,$mode_prereq \
+	"delayed checkout with $mode-collision don't write to the wrong place" '
+		test_config_global filter.delay.process \
+			"\"$TEST_ROOT/rot13-filter.pl\" --always-delay delayed.log clean smudge delay" &&
+		test_config_global filter.delay.required true &&
+
+		git init $mode-collision &&
+		(
+			cd $mode-collision &&
+			mkdir target-dir &&
+
+			empty_oid=$(printf "" | git hash-object -w --stdin) &&
+			symlink_oid=$(printf "%s" "$PWD/target-dir" | git hash-object -w --stdin) &&
+			attr_oid=$(echo "$dir/z filter=delay" | git hash-object -w --stdin) &&
+
+			cat >objs <<-EOF &&
+			100644 blob $empty_oid	$dir/x
+			100644 blob $empty_oid	$dir/y
+			100644 blob $empty_oid	$dir/z
+			120000 blob $symlink_oid	$symlink
+			100644 blob $attr_oid	.gitattributes
+			EOF
+
+			git update-index --index-info <objs &&
+			git commit -m "test commit"
+		) &&
+
+		git clone $mode-collision $mode-collision-cloned &&
+		# Make sure z was really delayed
+		grep "IN: smudge $dir/z .* \\[DELAYED\\]" $mode-collision-cloned/delayed.log &&
+
+		# Should not create $dir/z at $symlink/z
+		test_path_is_missing $mode-collision/target-dir/z
+	'
+done
+
+test_expect_success PERL,SYMLINKS,CASE_INSENSITIVE_FS \
+"delayed checkout with submodule collision don't write to the wrong place" '
+	git init collision-with-submodule &&
+	(
+		cd collision-with-submodule &&
+		git config filter.delay.process "\"$TEST_ROOT/rot13-filter.pl\" --always-delay delayed.log clean smudge delay" &&
+		git config filter.delay.required true &&
+
+		# We need Git to treat the submodule "a" and the
+		# leading dir "A" as different paths in the index.
+		git config --local core.ignoreCase false &&
+
+		empty_oid=$(printf "" | git hash-object -w --stdin) &&
+		attr_oid=$(echo "A/B/y filter=delay" | git hash-object -w --stdin) &&
+		cat >objs <<-EOF &&
+		100644 blob $empty_oid	A/B/x
+		100644 blob $empty_oid	A/B/y
+		100644 blob $attr_oid	.gitattributes
+		EOF
+		git update-index --index-info <objs &&
+
+		git init a &&
+		mkdir target-dir &&
+		symlink_oid=$(printf "%s" "$PWD/target-dir" | git -C a hash-object -w --stdin) &&
+		echo "120000 blob $symlink_oid	b" >objs &&
+		git -C a update-index --index-info <objs &&
+		git -C a commit -m sub &&
+		git submodule add ./a &&
+		git commit -m super &&
+
+		git checkout --recurse-submodules . &&
+		grep "IN: smudge A/B/y .* \\[DELAYED\\]" delayed.log &&
+		test_path_is_missing target-dir/y
+	)
+'
+
 test_done

t/t0021/rot13-filter.pl

@@ -2,9 +2,15 @@
 # Example implementation for the Git filter protocol version 2
 # See Documentation/gitattributes.txt, section "Filter Protocol"
 #
-# The first argument defines a debug log file that the script write to.
-# All remaining arguments define a list of supported protocol
-# capabilities ("clean", "smudge", etc).
+# Usage: rot13-filter.pl [--always-delay] <log path> <capabilities>
+#
+# Log path defines a debug log file that the script writes to. The
+# subsequent arguments define a list of supported protocol capabilities
+# ("clean", "smudge", etc).
+#
+# When --always-delay is given all pathnames with the "can-delay" flag
+# that don't appear on the list bellow are delayed with a count of 1
+# (see more below).
 #
 # This implementation supports special test cases:
 # (1) If data with the pathname "clean-write-fail.r" is processed with
@@ -53,6 +59,13 @@ sub gitperllib {
 use Git::Packet;
 
 my $MAX_PACKET_CONTENT_SIZE = 65516;
+
+my $always_delay = 0;
+if ( $ARGV[0] eq '--always-delay' ) {
+	$always_delay = 1;
+	shift @ARGV;
+}
+
 my $log_file                = shift @ARGV;
 my @capabilities            = @ARGV;
 
@@ -134,6 +147,8 @@ sub rot13 {
 			if ( $buffer eq "can-delay=1" ) {
 				if ( exists $DELAY{$pathname} and $DELAY{$pathname}{"requested"} == 0 ) {
 					$DELAY{$pathname}{"requested"} = 1;
+				} elsif ( !exists $DELAY{$pathname} and $always_delay ) {
+					$DELAY{$pathname} = { "requested" => 1, "count" => 1 };
 				}
 			} else {
 				die "Unknown message '$buffer'";

t/t2006-checkout-index-basic.sh

@@ -21,4 +21,50 @@ test_expect_success 'checkout-index -h in broken repository' '
 	test_i18ngrep "[Uu]sage" broken/usage
 '
 
+for mode in 'case' 'utf-8'
+do
+	case "$mode" in
+	case)	dir='A' symlink='a' mode_prereq='CASE_INSENSITIVE_FS' ;;
+	utf-8)
+		dir=$(printf "\141\314\210") symlink=$(printf "\303\244")
+		mode_prereq='UTF8_NFD_TO_NFC' ;;
+	esac
+
+	test_expect_success SYMLINKS,$mode_prereq \
+	"checkout-index with $mode-collision don't write to the wrong place" '
+		git init $mode-collision &&
+		(
+			cd $mode-collision &&
+			mkdir target-dir &&
+
+			empty_obj_hex=$(git hash-object -w --stdin </dev/null) &&
+			symlink_hex=$(printf "%s" "$PWD/target-dir" | git hash-object -w --stdin) &&
+
+			cat >objs <<-EOF &&
+			100644 blob ${empty_obj_hex}	${dir}/x
+			100644 blob ${empty_obj_hex}	${dir}/y
+			100644 blob ${empty_obj_hex}	${dir}/z
+			120000 blob ${symlink_hex}	${symlink}
+			EOF
+
+			git update-index --index-info <objs &&
+
+			# Note: the order is important here to exercise the
+			# case where the file at ${dir} has its type changed by
+			# the time Git tries to check out ${dir}/z.
+			#
+			# Also, we use core.precomposeUnicode=false because we
+			# want Git to treat the UTF-8 paths transparently on
+			# Mac OS, matching what is in the index.
+			#
+			git -c core.precomposeUnicode=false checkout-index -f \
+				${dir}/x ${dir}/y ${symlink} ${dir}/z &&
+
+			# Should not create ${dir}/z at ${symlink}/z
+			test_path_is_missing target-dir/z
+
+		)
+	'
+done
+
 test_done

unpack-trees.c

@@ -395,6 +395,9 @@ static int check_updates(struct unpack_trees_options *o)
 
 	progress = get_progress(o);
 
+	/* Start with clean cache to avoid using any possibly outdated info. */
+	invalidate_lstat_cache();
+
 	if (o->update)
 		git_attr_set_direction(GIT_ATTR_CHECKOUT);