From 283174b214c4c5416772dcacbc6389dd192969a0 Mon Sep 17 00:00:00 2001 From: John Cai Date: Fri, 12 May 2023 21:34:40 +0000 Subject: [PATCH 1/3] docs: clarify git-pack-refs --all will pack all refs --all packs not just branch tips but anything under refs/ with the exception of hidden refs and broken refs. Clarify this in the documentation. Signed-off-by: John Cai Signed-off-by: Junio C Hamano --- Documentation/git-pack-refs.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Documentation/git-pack-refs.txt b/Documentation/git-pack-refs.txt index 154081f2de2..22f00665006 100644 --- a/Documentation/git-pack-refs.txt +++ b/Documentation/git-pack-refs.txt @@ -51,8 +51,9 @@ The command by default packs all tags and refs that are already packed, and leaves other refs alone. This is because branches are expected to be actively developed and packing their tips does not help performance. -This option causes branch tips to be packed as well. Useful for -a repository with many branches of historical interests. +This option causes all refs to be packed as well, with the exception +of hidden refs, broken refs, and symbolic refs. Useful for a repository +with many branches of historical interests. --no-prune:: From 826ae79fca263bc2b70c54fddacb1603c5ebb9c6 Mon Sep 17 00:00:00 2001 From: John Cai Date: Fri, 12 May 2023 21:34:41 +0000 Subject: [PATCH 2/3] pack-refs: teach --exclude option to exclude refs from being packed At GitLab, we have a system that creates ephemeral internal refs that don't live long before getting deleted. Having an option to exclude certain refs from a packed-refs file allows these internal references to be deleted much more efficiently. Add an --exclude option to the pack-refs builtin, and use the ref exclusions API to exclude certain refs from being packed into the final packed-refs file Signed-off-by: John Cai Signed-off-by: Junio C Hamano --- Documentation/git-pack-refs.txt | 12 +++++++++++- builtin/pack-refs.c | 20 ++++++++++++++++---- refs.c | 4 ++-- refs.h | 7 ++++++- refs/debug.c | 4 ++-- refs/files-backend.c | 16 ++++++++++------ refs/packed-backend.c | 2 +- refs/refs-internal.h | 3 ++- revision.h | 2 +- t/helper/test-ref-store.c | 3 ++- t/t3210-pack-refs.sh | 16 ++++++++++++++++ 11 files changed, 69 insertions(+), 20 deletions(-) diff --git a/Documentation/git-pack-refs.txt b/Documentation/git-pack-refs.txt index 22f00665006..546aa122dff 100644 --- a/Documentation/git-pack-refs.txt +++ b/Documentation/git-pack-refs.txt @@ -8,7 +8,7 @@ git-pack-refs - Pack heads and tags for efficient repository access SYNOPSIS -------- [verse] -'git pack-refs' [--all] [--no-prune] +'git pack-refs' [--all] [--no-prune] [--exclude ] DESCRIPTION ----------- @@ -60,6 +60,16 @@ with many branches of historical interests. The command usually removes loose refs under `$GIT_DIR/refs` hierarchy after packing them. This option tells it not to. +--exclude :: + +Do not pack refs matching the given `glob(7)` pattern. Repetitions of this option +accumulate exclusion patterns. Use `--no-exclude` to clear and reset the list of +patterns. If a ref is already packed, including it with `--exclude` will not +unpack it. + +When used with `--all`, pack only loose refs which do not match any of +the provided `--exclude` patterns. + BUGS ---- diff --git a/builtin/pack-refs.c b/builtin/pack-refs.c index 9833815fb30..1d1a64fe386 100644 --- a/builtin/pack-refs.c +++ b/builtin/pack-refs.c @@ -4,22 +4,34 @@ #include "parse-options.h" #include "refs.h" #include "repository.h" +#include "revision.h" static char const * const pack_refs_usage[] = { - N_("git pack-refs [--all] [--no-prune]"), + N_("git pack-refs [--all] [--no-prune] [--exclude ]"), NULL }; int cmd_pack_refs(int argc, const char **argv, const char *prefix) { unsigned int flags = PACK_REFS_PRUNE; + static struct ref_exclusions excludes = REF_EXCLUSIONS_INIT; + struct pack_refs_opts pack_refs_opts = {.exclusions = &excludes, .flags = flags}; + static struct string_list option_excluded_refs = STRING_LIST_INIT_NODUP; + struct string_list_item *item; + struct option opts[] = { - OPT_BIT(0, "all", &flags, N_("pack everything"), PACK_REFS_ALL), - OPT_BIT(0, "prune", &flags, N_("prune loose refs (default)"), PACK_REFS_PRUNE), + OPT_BIT(0, "all", &pack_refs_opts.flags, N_("pack everything"), PACK_REFS_ALL), + OPT_BIT(0, "prune", &pack_refs_opts.flags, N_("prune loose refs (default)"), PACK_REFS_PRUNE), + OPT_STRING_LIST(0, "exclude", &option_excluded_refs, N_("pattern"), + N_("references to exclude")), OPT_END(), }; git_config(git_default_config, NULL); if (parse_options(argc, argv, prefix, opts, pack_refs_usage, 0)) usage_with_options(pack_refs_usage, opts); - return refs_pack_refs(get_main_ref_store(the_repository), flags); + + for_each_string_list_item(item, &option_excluded_refs) + add_ref_exclusion(pack_refs_opts.exclusions, item->string); + + return refs_pack_refs(get_main_ref_store(the_repository), &pack_refs_opts); } diff --git a/refs.c b/refs.c index d2a98e1c21f..881a0da65cf 100644 --- a/refs.c +++ b/refs.c @@ -2132,9 +2132,9 @@ void base_ref_store_init(struct ref_store *refs, struct repository *repo, } /* backend functions */ -int refs_pack_refs(struct ref_store *refs, unsigned int flags) +int refs_pack_refs(struct ref_store *refs, struct pack_refs_opts *opts) { - return refs->be->pack_refs(refs, flags); + return refs->be->pack_refs(refs, opts); } int peel_iterated_oid(const struct object_id *base, struct object_id *peeled) diff --git a/refs.h b/refs.h index 123cfa44244..46020bd335c 100644 --- a/refs.h +++ b/refs.h @@ -63,6 +63,11 @@ struct worktree; #define RESOLVE_REF_NO_RECURSE 0x02 #define RESOLVE_REF_ALLOW_BAD_NAME 0x04 +struct pack_refs_opts { + unsigned int flags; + struct ref_exclusions *exclusions; +}; + const char *refs_resolve_ref_unsafe(struct ref_store *refs, const char *refname, int resolve_flags, @@ -405,7 +410,7 @@ void warn_dangling_symrefs(FILE *fp, const char *msg_fmt, * Write a packed-refs file for the current repository. * flags: Combination of the above PACK_REFS_* flags. */ -int refs_pack_refs(struct ref_store *refs, unsigned int flags); +int refs_pack_refs(struct ref_store *refs, struct pack_refs_opts *opts); /* * Setup reflog before using. Fill in err and return -1 on failure. diff --git a/refs/debug.c b/refs/debug.c index adc34c836fc..63f434ea26b 100644 --- a/refs/debug.c +++ b/refs/debug.c @@ -122,10 +122,10 @@ static int debug_initial_transaction_commit(struct ref_store *refs, return res; } -static int debug_pack_refs(struct ref_store *ref_store, unsigned int flags) +static int debug_pack_refs(struct ref_store *ref_store, struct pack_refs_opts *opts) { struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store; - int res = drefs->refs->be->pack_refs(drefs->refs, flags); + int res = drefs->refs->be->pack_refs(drefs->refs, opts); trace_printf_key(&trace_refs, "pack_refs: %d\n", res); return res; } diff --git a/refs/files-backend.c b/refs/files-backend.c index d0581ee41ac..8620e013625 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -19,6 +19,7 @@ #include "../worktree.h" #include "../wrapper.h" #include "../write-or-die.h" +#include "../revision.h" /* * This backend uses the following flags in `ref_update::flags` for @@ -1173,15 +1174,18 @@ static void prune_refs(struct files_ref_store *refs, struct ref_to_prune **refs_ */ static int should_pack_ref(const char *refname, const struct object_id *oid, unsigned int ref_flags, - unsigned int pack_flags) + struct pack_refs_opts *opts) { /* Do not pack per-worktree refs: */ if (parse_worktree_ref(refname, NULL, NULL, NULL) != REF_WORKTREE_SHARED) return 0; + if (ref_excluded(opts->exclusions, refname)) + return 0; + /* Do not pack non-tags unless PACK_REFS_ALL is set: */ - if (!(pack_flags & PACK_REFS_ALL) && !starts_with(refname, "refs/tags/")) + if (!(opts->flags & PACK_REFS_ALL) && !starts_with(refname, "refs/tags/")) return 0; /* Do not pack symbolic refs: */ @@ -1195,7 +1199,8 @@ static int should_pack_ref(const char *refname, return 1; } -static int files_pack_refs(struct ref_store *ref_store, unsigned int flags) +static int files_pack_refs(struct ref_store *ref_store, + struct pack_refs_opts *opts) { struct files_ref_store *refs = files_downcast(ref_store, REF_STORE_WRITE | REF_STORE_ODB, @@ -1220,8 +1225,7 @@ static int files_pack_refs(struct ref_store *ref_store, unsigned int flags) * in the packed ref cache. If the reference should be * pruned, also add it to refs_to_prune. */ - if (!should_pack_ref(iter->refname, iter->oid, iter->flags, - flags)) + if (!should_pack_ref(iter->refname, iter->oid, iter->flags, opts)) continue; /* @@ -1235,7 +1239,7 @@ static int files_pack_refs(struct ref_store *ref_store, unsigned int flags) iter->refname, err.buf); /* Schedule the loose reference for pruning if requested. */ - if ((flags & PACK_REFS_PRUNE)) { + if ((opts->flags & PACK_REFS_PRUNE)) { struct ref_to_prune *n; FLEX_ALLOC_STR(n, name, iter->refname); oidcpy(&n->oid, iter->oid); diff --git a/refs/packed-backend.c b/refs/packed-backend.c index 34c0c4e20f1..87ba5f520e4 100644 --- a/refs/packed-backend.c +++ b/refs/packed-backend.c @@ -1576,7 +1576,7 @@ static int packed_delete_refs(struct ref_store *ref_store, const char *msg, } static int packed_pack_refs(struct ref_store *ref_store UNUSED, - unsigned int flags UNUSED) + struct pack_refs_opts *pack_opts UNUSED) { /* * Packed refs are already packed. It might be that loose refs diff --git a/refs/refs-internal.h b/refs/refs-internal.h index a85d113123c..f72b7be8941 100644 --- a/refs/refs-internal.h +++ b/refs/refs-internal.h @@ -547,7 +547,8 @@ typedef int ref_transaction_commit_fn(struct ref_store *refs, struct ref_transaction *transaction, struct strbuf *err); -typedef int pack_refs_fn(struct ref_store *ref_store, unsigned int flags); +typedef int pack_refs_fn(struct ref_store *ref_store, + struct pack_refs_opts *opts); typedef int create_symref_fn(struct ref_store *ref_store, const char *ref_target, const char *refs_heads_master, diff --git a/revision.h b/revision.h index e8f6de96849..ac7b08bbfd8 100644 --- a/revision.h +++ b/revision.h @@ -86,7 +86,7 @@ struct rev_cmdline_info { struct ref_exclusions { /* * Excluded refs is a list of wildmatch patterns. If any of the - * patterns matches, the reference will be excluded. + * patterns match, the reference will be excluded. */ struct string_list excluded_refs; diff --git a/t/helper/test-ref-store.c b/t/helper/test-ref-store.c index 6d8f844e9c7..de4197708d9 100644 --- a/t/helper/test-ref-store.c +++ b/t/helper/test-ref-store.c @@ -116,8 +116,9 @@ static struct flag_definition pack_flags[] = { FLAG_DEF(PACK_REFS_PRUNE), static int cmd_pack_refs(struct ref_store *refs, const char **argv) { unsigned int flags = arg_flags(*argv++, "flags", pack_flags); + struct pack_refs_opts pack_opts = { .flags = flags }; - return refs_pack_refs(refs, flags); + return refs_pack_refs(refs, &pack_opts); } static int cmd_create_symref(struct ref_store *refs, const char **argv) diff --git a/t/t3210-pack-refs.sh b/t/t3210-pack-refs.sh index 07a0ff93def..925b90cd3ba 100755 --- a/t/t3210-pack-refs.sh +++ b/t/t3210-pack-refs.sh @@ -108,6 +108,22 @@ test_expect_success \ git branch -d n/o/p && git branch n' +test_expect_success 'test excluded refs are not packed' ' + git branch dont_pack1 && + git branch dont_pack2 && + git branch pack_this && + git pack-refs --all --exclude "refs/heads/dont_pack*" && + test -f .git/refs/heads/dont_pack1 && + test -f .git/refs/heads/dont_pack2 && + ! test -f .git/refs/heads/pack_this' + +test_expect_success 'test --no-exclude refs clears excluded refs' ' + git branch dont_pack3 && + git branch dont_pack4 && + git pack-refs --all --exclude "refs/heads/dont_pack*" --no-exclude && + ! test -f .git/refs/heads/dont_pack3 && + ! test -f .git/refs/heads/dont_pack4' + test_expect_success \ 'see if up-to-date packed refs are preserved' \ 'git branch q && From 4fe42f326e10a547dc65dfe9e5ceaeeee02b98db Mon Sep 17 00:00:00 2001 From: John Cai Date: Fri, 12 May 2023 21:34:42 +0000 Subject: [PATCH 3/3] pack-refs: teach pack-refs --include option Allow users to be more selective over which refs to pack by adding an --include option to git-pack-refs. The existing options allow some measure of selectivity. By default git-pack-refs packs all tags. --all can be used to include all refs, and the previous commit added the ability to exclude certain refs with --exclude. While these knobs give the user some selection over which refs to pack, it could be useful to give more control. For instance, a repository may have a set of branches that are rarely updated and would benefit from being packed. --include would allow the user to easily include a set of branches to be packed while leaving everything else unpacked. Signed-off-by: John Cai Signed-off-by: Junio C Hamano --- Documentation/git-pack-refs.txt | 14 +++++++++++++- builtin/pack-refs.c | 15 +++++++++++++-- refs.h | 1 + refs/files-backend.c | 18 ++++++++++-------- t/helper/test-ref-store.c | 10 +++++++++- t/t3210-pack-refs.sh | 21 +++++++++++++++++++++ 6 files changed, 67 insertions(+), 12 deletions(-) diff --git a/Documentation/git-pack-refs.txt b/Documentation/git-pack-refs.txt index 546aa122dff..284956acb3c 100644 --- a/Documentation/git-pack-refs.txt +++ b/Documentation/git-pack-refs.txt @@ -8,7 +8,7 @@ git-pack-refs - Pack heads and tags for efficient repository access SYNOPSIS -------- [verse] -'git pack-refs' [--all] [--no-prune] [--exclude ] +'git pack-refs' [--all] [--no-prune] [--include ] [--exclude ] DESCRIPTION ----------- @@ -60,6 +60,15 @@ with many branches of historical interests. The command usually removes loose refs under `$GIT_DIR/refs` hierarchy after packing them. This option tells it not to. +--include :: + +Pack refs based on a `glob(7)` pattern. Repetitions of this option +accumulate inclusion patterns. If a ref is both included in `--include` and +`--exclude`, `--exclude` takes precedence. Using `--include` will preclude all +tags from being included by default. Symbolic refs and broken refs will never +be packed. When used with `--all`, it will be a noop. Use `--no-include` to clear +and reset the list of patterns. + --exclude :: Do not pack refs matching the given `glob(7)` pattern. Repetitions of this option @@ -70,6 +79,9 @@ unpack it. When used with `--all`, pack only loose refs which do not match any of the provided `--exclude` patterns. +When used with `--include`, refs provided to `--include`, minus refs that are +provided to `--exclude` will be packed. + BUGS ---- diff --git a/builtin/pack-refs.c b/builtin/pack-refs.c index 1d1a64fe386..bcf383cac9d 100644 --- a/builtin/pack-refs.c +++ b/builtin/pack-refs.c @@ -7,7 +7,7 @@ #include "revision.h" static char const * const pack_refs_usage[] = { - N_("git pack-refs [--all] [--no-prune] [--exclude ]"), + N_("git pack-refs [--all] [--no-prune] [--include ] [--exclude ]"), NULL }; @@ -15,13 +15,18 @@ int cmd_pack_refs(int argc, const char **argv, const char *prefix) { unsigned int flags = PACK_REFS_PRUNE; static struct ref_exclusions excludes = REF_EXCLUSIONS_INIT; - struct pack_refs_opts pack_refs_opts = {.exclusions = &excludes, .flags = flags}; + static struct string_list included_refs = STRING_LIST_INIT_NODUP; + struct pack_refs_opts pack_refs_opts = { .exclusions = &excludes, + .includes = &included_refs, + .flags = flags }; static struct string_list option_excluded_refs = STRING_LIST_INIT_NODUP; struct string_list_item *item; struct option opts[] = { OPT_BIT(0, "all", &pack_refs_opts.flags, N_("pack everything"), PACK_REFS_ALL), OPT_BIT(0, "prune", &pack_refs_opts.flags, N_("prune loose refs (default)"), PACK_REFS_PRUNE), + OPT_STRING_LIST(0, "include", pack_refs_opts.includes, N_("pattern"), + N_("references to include")), OPT_STRING_LIST(0, "exclude", &option_excluded_refs, N_("pattern"), N_("references to exclude")), OPT_END(), @@ -33,5 +38,11 @@ int cmd_pack_refs(int argc, const char **argv, const char *prefix) for_each_string_list_item(item, &option_excluded_refs) add_ref_exclusion(pack_refs_opts.exclusions, item->string); + if (pack_refs_opts.flags & PACK_REFS_ALL) + string_list_append(pack_refs_opts.includes, "*"); + + if (!pack_refs_opts.includes->nr) + string_list_append(pack_refs_opts.includes, "refs/tags/*"); + return refs_pack_refs(get_main_ref_store(the_repository), &pack_refs_opts); } diff --git a/refs.h b/refs.h index 46020bd335c..933fdebe584 100644 --- a/refs.h +++ b/refs.h @@ -66,6 +66,7 @@ struct worktree; struct pack_refs_opts { unsigned int flags; struct ref_exclusions *exclusions; + struct string_list *includes; }; const char *refs_resolve_ref_unsafe(struct ref_store *refs, diff --git a/refs/files-backend.c b/refs/files-backend.c index 8620e013625..fcaf842ed43 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -1176,18 +1176,13 @@ static int should_pack_ref(const char *refname, const struct object_id *oid, unsigned int ref_flags, struct pack_refs_opts *opts) { + struct string_list_item *item; + /* Do not pack per-worktree refs: */ if (parse_worktree_ref(refname, NULL, NULL, NULL) != REF_WORKTREE_SHARED) return 0; - if (ref_excluded(opts->exclusions, refname)) - return 0; - - /* Do not pack non-tags unless PACK_REFS_ALL is set: */ - if (!(opts->flags & PACK_REFS_ALL) && !starts_with(refname, "refs/tags/")) - return 0; - /* Do not pack symbolic refs: */ if (ref_flags & REF_ISSYMREF) return 0; @@ -1196,7 +1191,14 @@ static int should_pack_ref(const char *refname, if (!ref_resolves_to_object(refname, the_repository, oid, ref_flags)) return 0; - return 1; + if (ref_excluded(opts->exclusions, refname)) + return 0; + + for_each_string_list_item(item, opts->includes) + if (!wildmatch(item->string, refname, 0)) + return 1; + + return 0; } static int files_pack_refs(struct ref_store *ref_store, diff --git a/t/helper/test-ref-store.c b/t/helper/test-ref-store.c index de4197708d9..a6977b5e839 100644 --- a/t/helper/test-ref-store.c +++ b/t/helper/test-ref-store.c @@ -5,6 +5,7 @@ #include "worktree.h" #include "object-store.h" #include "repository.h" +#include "revision.h" struct flag_definition { const char *name; @@ -116,7 +117,14 @@ static struct flag_definition pack_flags[] = { FLAG_DEF(PACK_REFS_PRUNE), static int cmd_pack_refs(struct ref_store *refs, const char **argv) { unsigned int flags = arg_flags(*argv++, "flags", pack_flags); - struct pack_refs_opts pack_opts = { .flags = flags }; + static struct ref_exclusions exclusions = REF_EXCLUSIONS_INIT; + static struct string_list included_refs = STRING_LIST_INIT_NODUP; + struct pack_refs_opts pack_opts = { .flags = flags, + .exclusions = &exclusions, + .includes = &included_refs }; + + if (pack_opts.flags & PACK_REFS_ALL) + string_list_append(pack_opts.includes, "*"); return refs_pack_refs(refs, &pack_opts); } diff --git a/t/t3210-pack-refs.sh b/t/t3210-pack-refs.sh index 925b90cd3ba..9f399d2f75a 100755 --- a/t/t3210-pack-refs.sh +++ b/t/t3210-pack-refs.sh @@ -124,6 +124,27 @@ test_expect_success 'test --no-exclude refs clears excluded refs' ' ! test -f .git/refs/heads/dont_pack3 && ! test -f .git/refs/heads/dont_pack4' +test_expect_success 'test only included refs are packed' ' + git branch pack_this1 && + git branch pack_this2 && + git tag dont_pack5 && + git pack-refs --include "refs/heads/pack_this*" && + test -f .git/refs/tags/dont_pack5 && + ! test -f .git/refs/heads/pack_this1 && + ! test -f .git/refs/heads/pack_this2' + +test_expect_success 'test --no-include refs clears included refs' ' + git branch pack1 && + git branch pack2 && + git pack-refs --include "refs/heads/pack*" --no-include && + test -f .git/refs/heads/pack1 && + test -f .git/refs/heads/pack2' + +test_expect_success 'test --exclude takes precedence over --include' ' + git branch dont_pack5 && + git pack-refs --include "refs/heads/pack*" --exclude "refs/heads/pack*" && + test -f .git/refs/heads/dont_pack5' + test_expect_success \ 'see if up-to-date packed refs are preserved' \ 'git branch q &&