diff --git a/Documentation/config/pack.txt b/Documentation/config/pack.txt index 53093d9996..d4c7c9d4e4 100644 --- a/Documentation/config/pack.txt +++ b/Documentation/config/pack.txt @@ -171,9 +171,15 @@ pack.writeBitmapLookupTable:: beneficial in repositories that have relatively large bitmap indexes. Defaults to false. +pack.readReverseIndex:: + When true, git will read any .rev file(s) that may be available + (see: linkgit:gitformat-pack[5]). When false, the reverse index + will be generated from scratch and stored in memory. Defaults to + true. + pack.writeReverseIndex:: When true, git will write a corresponding .rev file (see: linkgit:gitformat-pack[5]) for each new packfile that it writes in all places except for linkgit:git-fast-import[1] and in the bulk checkin mechanism. - Defaults to false. + Defaults to true. diff --git a/builtin/index-pack.c b/builtin/index-pack.c index 0f59cf8aa0..bb67e16655 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -1756,12 +1756,13 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix) fsck_options.walk = mark_link; reset_pack_idx_option(&opts); + opts.flags |= WRITE_REV; git_config(git_index_pack_config, &opts); if (prefix && chdir(prefix)) die(_("Cannot come back to cwd")); - if (git_env_bool(GIT_TEST_WRITE_REV_INDEX, 0)) - rev_index = 1; + if (git_env_bool(GIT_TEST_NO_WRITE_REV_INDEX, 0)) + rev_index = 0; else rev_index = !!(opts.flags & (WRITE_REV_VERIFY | WRITE_REV)); diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index bbba1af27b..a5b466839b 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -4294,9 +4294,10 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) } reset_pack_idx_option(&pack_idx_opts); + pack_idx_opts.flags |= WRITE_REV; git_config(git_pack_config, NULL); - if (git_env_bool(GIT_TEST_WRITE_REV_INDEX, 0)) - pack_idx_opts.flags |= WRITE_REV; + if (git_env_bool(GIT_TEST_NO_WRITE_REV_INDEX, 0)) + pack_idx_opts.flags &= ~WRITE_REV; progress = isatty(2); argc = parse_options(argc, argv, prefix, pack_objects_options, diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index b098e10f52..a18b13a41d 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -27,7 +27,7 @@ linux-TEST-vars) export GIT_TEST_MULTI_PACK_INDEX=1 export GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=1 export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master - export GIT_TEST_WRITE_REV_INDEX=1 + export GIT_TEST_NO_WRITE_REV_INDEX=1 export GIT_TEST_CHECKOUT_WORKERS=2 ;; linux-clang) diff --git a/pack-bitmap.c b/pack-bitmap.c index 295b8abc19..cd5f472488 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -465,7 +465,7 @@ static int open_pack_bitmap_1(struct bitmap_index *bitmap_git, struct packed_git return 0; } -static int load_reverse_index(struct bitmap_index *bitmap_git) +static int load_reverse_index(struct repository *r, struct bitmap_index *bitmap_git) { if (bitmap_is_midx(bitmap_git)) { uint32_t i; @@ -479,23 +479,23 @@ static int load_reverse_index(struct bitmap_index *bitmap_git) * since we will need to make use of them in pack-objects. */ for (i = 0; i < bitmap_git->midx->num_packs; i++) { - ret = load_pack_revindex(bitmap_git->midx->packs[i]); + ret = load_pack_revindex(r, bitmap_git->midx->packs[i]); if (ret) return ret; } return 0; } - return load_pack_revindex(bitmap_git->pack); + return load_pack_revindex(r, bitmap_git->pack); } -static int load_bitmap(struct bitmap_index *bitmap_git) +static int load_bitmap(struct repository *r, struct bitmap_index *bitmap_git) { assert(bitmap_git->map); bitmap_git->bitmaps = kh_init_oid_map(); bitmap_git->ext_index.positions = kh_init_oid_pos(); - if (load_reverse_index(bitmap_git)) + if (load_reverse_index(r, bitmap_git)) goto failed; if (!(bitmap_git->commits = read_bitmap_1(bitmap_git)) || @@ -582,7 +582,7 @@ struct bitmap_index *prepare_bitmap_git(struct repository *r) { struct bitmap_index *bitmap_git = xcalloc(1, sizeof(*bitmap_git)); - if (!open_bitmap(r, bitmap_git) && !load_bitmap(bitmap_git)) + if (!open_bitmap(r, bitmap_git) && !load_bitmap(r, bitmap_git)) return bitmap_git; free_bitmap_index(bitmap_git); @@ -591,9 +591,10 @@ struct bitmap_index *prepare_bitmap_git(struct repository *r) struct bitmap_index *prepare_midx_bitmap_git(struct multi_pack_index *midx) { + struct repository *r = the_repository; struct bitmap_index *bitmap_git = xcalloc(1, sizeof(*bitmap_git)); - if (!open_midx_bitmap_1(bitmap_git, midx) && !load_bitmap(bitmap_git)) + if (!open_midx_bitmap_1(bitmap_git, midx) && !load_bitmap(r, bitmap_git)) return bitmap_git; free_bitmap_index(bitmap_git); @@ -1594,7 +1595,7 @@ struct bitmap_index *prepare_bitmap_walk(struct rev_info *revs, * from disk. this is the point of no return; after this the rev_list * becomes invalidated and we must perform the revwalk through bitmaps */ - if (load_bitmap(bitmap_git) < 0) + if (load_bitmap(revs->repo, bitmap_git) < 0) goto cleanup; object_array_clear(&revs->pending); @@ -1744,6 +1745,7 @@ int reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git, uint32_t *entries, struct bitmap **reuse_out) { + struct repository *r = the_repository; struct packed_git *pack; struct bitmap *result = bitmap_git->result; struct bitmap *reuse; @@ -1754,7 +1756,7 @@ int reuse_partial_packfile_from_bitmap(struct bitmap_index *bitmap_git, assert(result); - load_reverse_index(bitmap_git); + load_reverse_index(r, bitmap_git); if (bitmap_is_midx(bitmap_git)) pack = bitmap_git->midx->packs[midx_preferred_pack(bitmap_git)]; @@ -2134,11 +2136,12 @@ int rebuild_bitmap(const uint32_t *reposition, uint32_t *create_bitmap_mapping(struct bitmap_index *bitmap_git, struct packing_data *mapping) { + struct repository *r = the_repository; uint32_t i, num_objects; uint32_t *reposition; if (!bitmap_is_midx(bitmap_git)) - load_reverse_index(bitmap_git); + load_reverse_index(r, bitmap_git); else if (load_midx_revindex(bitmap_git->midx) < 0) BUG("rebuild_existing_bitmaps: missing required rev-cache " "extension"); diff --git a/pack-revindex.c b/pack-revindex.c index 4d9bb41b4d..db282dac8d 100644 --- a/pack-revindex.c +++ b/pack-revindex.c @@ -207,6 +207,9 @@ static int load_revindex_from_disk(char *revindex_name, size_t revindex_size; struct revindex_header *hdr; + if (git_env_bool(GIT_TEST_REV_INDEX_DIE_ON_DISK, 0)) + die("dying as requested by '%s'", GIT_TEST_REV_INDEX_DIE_ON_DISK); + fd = git_open(revindex_name); if (fd < 0) { @@ -285,12 +288,15 @@ static int load_pack_revindex_from_disk(struct packed_git *p) return ret; } -int load_pack_revindex(struct packed_git *p) +int load_pack_revindex(struct repository *r, struct packed_git *p) { if (p->revindex || p->revindex_data) return 0; - if (!load_pack_revindex_from_disk(p)) + prepare_repo_settings(r); + + if (r->settings.pack_read_reverse_index && + !load_pack_revindex_from_disk(p)) return 0; else if (!create_pack_revindex_in_memory(p)) return 0; @@ -358,7 +364,7 @@ int offset_to_pack_pos(struct packed_git *p, off_t ofs, uint32_t *pos) { unsigned lo, hi; - if (load_pack_revindex(p) < 0) + if (load_pack_revindex(the_repository, p) < 0) return -1; lo = 0; diff --git a/pack-revindex.h b/pack-revindex.h index 4974e75eb4..46e834064e 100644 --- a/pack-revindex.h +++ b/pack-revindex.h @@ -34,11 +34,13 @@ #define RIDX_SIGNATURE 0x52494458 /* "RIDX" */ #define RIDX_VERSION 1 -#define GIT_TEST_WRITE_REV_INDEX "GIT_TEST_WRITE_REV_INDEX" +#define GIT_TEST_NO_WRITE_REV_INDEX "GIT_TEST_NO_WRITE_REV_INDEX" #define GIT_TEST_REV_INDEX_DIE_IN_MEMORY "GIT_TEST_REV_INDEX_DIE_IN_MEMORY" +#define GIT_TEST_REV_INDEX_DIE_ON_DISK "GIT_TEST_REV_INDEX_DIE_ON_DISK" struct packed_git; struct multi_pack_index; +struct repository; /* * load_pack_revindex populates the revindex's internal data-structures for the @@ -47,7 +49,7 @@ struct multi_pack_index; * If a '.rev' file is present it is mmap'd, and pointers are assigned into it * (instead of using the in-memory variant). */ -int load_pack_revindex(struct packed_git *p); +int load_pack_revindex(struct repository *r, struct packed_git *p); /* * load_midx_revindex loads the '.rev' file corresponding to the given diff --git a/pack-write.c b/pack-write.c index 9c37121be3..381555359c 100644 --- a/pack-write.c +++ b/pack-write.c @@ -570,6 +570,8 @@ void stage_tmp_packfiles(struct strbuf *name_buffer, rename_tmp_packfile(name_buffer, rev_tmp_name, "rev"); if (mtimes_tmp_name) rename_tmp_packfile(name_buffer, mtimes_tmp_name, "mtimes"); + + free((char *)rev_tmp_name); } void write_promisor_file(const char *promisor_name, struct ref **sought, int nr_sought) diff --git a/packfile.c b/packfile.c index 9ae2278c22..57df0c1956 100644 --- a/packfile.c +++ b/packfile.c @@ -2154,7 +2154,7 @@ int for_each_object_in_pack(struct packed_git *p, int r = 0; if (flags & FOR_EACH_OBJECT_PACK_ORDER) { - if (load_pack_revindex(p)) + if (load_pack_revindex(the_repository, p)) return -1; } diff --git a/repo-settings.c b/repo-settings.c index 0a6c0b381f..bdd7640ab0 100644 --- a/repo-settings.c +++ b/repo-settings.c @@ -63,6 +63,7 @@ void prepare_repo_settings(struct repository *r) repo_cfg_bool(r, "core.multipackindex", &r->settings.core_multi_pack_index, 1); repo_cfg_bool(r, "index.sparse", &r->settings.sparse_index, 0); repo_cfg_bool(r, "index.skiphash", &r->settings.index_skip_hash, r->settings.index_skip_hash); + repo_cfg_bool(r, "pack.readreverseindex", &r->settings.pack_read_reverse_index, 1); /* * The GIT_TEST_MULTI_PACK_INDEX variable is special in that diff --git a/repository.h b/repository.h index 15a8afc5fb..ed73e799b6 100644 --- a/repository.h +++ b/repository.h @@ -37,6 +37,7 @@ struct repo_settings { int fetch_write_commit_graph; int command_requires_full_index; int sparse_index; + int pack_read_reverse_index; struct fsmonitor_settings *fsmonitor; /* lazily loaded */ diff --git a/t/README b/t/README index 29576c3748..bdfac4cceb 100644 --- a/t/README +++ b/t/README @@ -475,7 +475,7 @@ GIT_TEST_DEFAULT_HASH= specifies which hash algorithm to use in the test scripts. Recognized values for are "sha1" and "sha256". -GIT_TEST_WRITE_REV_INDEX=, when true enables the +GIT_TEST_NO_WRITE_REV_INDEX=, when true disables the 'pack.writeReverseIndex' setting. GIT_TEST_SPARSE_INDEX=, when true enables index writes to use the diff --git a/t/perf/p5312-pack-bitmaps-revs.sh b/t/perf/p5312-pack-bitmaps-revs.sh index 0684b690af..ceec60656b 100755 --- a/t/perf/p5312-pack-bitmaps-revs.sh +++ b/t/perf/p5312-pack-bitmaps-revs.sh @@ -12,8 +12,7 @@ test_lookup_pack_bitmap () { test_perf_large_repo test_expect_success 'setup bitmap config' ' - git config pack.writebitmaps true && - git config pack.writeReverseIndex true + git config pack.writebitmaps true ' # we need to create the tag up front such that it is covered by the repack and diff --git a/t/t5325-reverse-index.sh b/t/t5325-reverse-index.sh index d042d26f2b..0548fce1aa 100755 --- a/t/t5325-reverse-index.sh +++ b/t/t5325-reverse-index.sh @@ -1,17 +1,20 @@ #!/bin/sh test_description='on-disk reverse index' + +TEST_PASSES_SANITIZE_LEAK=true . ./test-lib.sh # The below tests want control over the 'pack.writeReverseIndex' setting # themselves to assert various combinations of it with other options. -sane_unset GIT_TEST_WRITE_REV_INDEX +sane_unset GIT_TEST_NO_WRITE_REV_INDEX packdir=.git/objects/pack test_expect_success 'setup' ' test_commit base && + test_config pack.writeReverseIndex false && pack=$(git pack-objects --all $packdir/pack) && rev=$packdir/pack-$pack.rev && @@ -94,6 +97,17 @@ test_expect_success 'reverse index is not generated when available on disk' ' --batch-check="%(objectsize:disk)" tip && + GIT_TEST_REV_INDEX_DIE_ON_DISK=1 git cat-file \ + --batch-check="%(objectsize:disk)"