From 41abfe15d95ede4c2a047180a6062eac23d8f7d6 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 9 Feb 2021 13:42:28 +0000 Subject: [PATCH 1/2] maintenance: add pack-refs task It is valuable to collect loose refs into a more compressed form. This is typically the packed-refs file, although this could be the reftable in the future. Having packed refs can be extremely valuable in repos with many tags or remote branches that are not modified by the local user, but still are necessary for other queries. For instance, with many exploded refs, commands such as git describe --tags --exact-match HEAD can be very slow (multiple seconds). This command in particular is used by terminal prompts to show when a detatched HEAD is pointing to an existing tag, so having it be slow causes significant delays for users. Add a new 'pack-refs' maintenance task. It runs 'git pack-refs --all --prune' to move loose refs into a packed form. For now, that is the packed-refs file, but could adjust to other file formats in the future. This is the first of several sub-tasks of the 'gc' task that could be extracted to their own tasks. In this process, we should not change the behavior of the 'gc' task since that remains the default way to keep repositories maintained. Creating a new task for one of these sub-tasks only provides more customization options for those choosing to not use the 'gc' task. It is certainly possible to have both the 'gc' and 'pack-refs' tasks enabled and run regularly. While they may repeat effort, they do not conflict in a destructive way. The 'auto_condition' function pointer is left NULL for now. We could extend this in the future to have a condition check if pack-refs should be run during 'git maintenance run --auto'. Signed-off-by: Derrick Stolee Reviewed-by: Taylor Blau Signed-off-by: Junio C Hamano --- Documentation/git-maintenance.txt | 6 ++++++ builtin/gc.c | 21 +++++++++++++++++---- t/t7900-maintenance.sh | 12 ++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/Documentation/git-maintenance.txt b/Documentation/git-maintenance.txt index 3b432171d6..80ddd33ceb 100644 --- a/Documentation/git-maintenance.txt +++ b/Documentation/git-maintenance.txt @@ -145,6 +145,12 @@ incremental-repack:: which is a special case that attempts to repack all pack-files into a single pack-file. +pack-refs:: + The `pack-refs` task collects the loose reference files and + collects them into a single file. This speeds up operations that + need to iterate across many references. See linkgit:git-pack-refs[1] + for more information. + OPTIONS ------- --auto:: diff --git a/builtin/gc.c b/builtin/gc.c index 4c40594d66..41bec4f177 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -54,7 +54,6 @@ static const char *prune_worktrees_expire = "3.months.ago"; static unsigned long big_pack_threshold; static unsigned long max_delta_cache_size = DEFAULT_DELTA_CACHE_SIZE; -static struct strvec pack_refs_cmd = STRVEC_INIT; static struct strvec reflog = STRVEC_INIT; static struct strvec repack = STRVEC_INIT; static struct strvec prune = STRVEC_INIT; @@ -163,6 +162,15 @@ static void gc_config(void) git_config(git_default_config, NULL); } +struct maintenance_run_opts; +static int maintenance_task_pack_refs(MAYBE_UNUSED struct maintenance_run_opts *opts) +{ + struct strvec pack_refs_cmd = STRVEC_INIT; + strvec_pushl(&pack_refs_cmd, "pack-refs", "--all", "--prune", NULL); + + return run_command_v_opt(pack_refs_cmd.v, RUN_GIT_CMD); +} + static int too_many_loose_objects(void) { /* @@ -518,8 +526,8 @@ static void gc_before_repack(void) if (done++) return; - if (pack_refs && run_command_v_opt(pack_refs_cmd.v, RUN_GIT_CMD)) - die(FAILED_RUN, pack_refs_cmd.v[0]); + if (pack_refs && maintenance_task_pack_refs(NULL)) + die(FAILED_RUN, "pack-refs"); if (prune_reflogs && run_command_v_opt(reflog.v, RUN_GIT_CMD)) die(FAILED_RUN, reflog.v[0]); @@ -556,7 +564,6 @@ int cmd_gc(int argc, const char **argv, const char *prefix) if (argc == 2 && !strcmp(argv[1], "-h")) usage_with_options(builtin_gc_usage, builtin_gc_options); - strvec_pushl(&pack_refs_cmd, "pack-refs", "--all", "--prune", NULL); strvec_pushl(&reflog, "reflog", "expire", "--all", NULL); strvec_pushl(&repack, "repack", "-d", "-l", NULL); strvec_pushl(&prune, "prune", "--expire", NULL); @@ -1224,6 +1231,7 @@ enum maintenance_task_label { TASK_INCREMENTAL_REPACK, TASK_GC, TASK_COMMIT_GRAPH, + TASK_PACK_REFS, /* Leave as final value */ TASK__COUNT @@ -1255,6 +1263,11 @@ static struct maintenance_task tasks[] = { maintenance_task_commit_graph, should_write_commit_graph, }, + [TASK_PACK_REFS] = { + "pack-refs", + maintenance_task_pack_refs, + NULL, + }, }; static int compare_tasks_by_selection(const void *a_, const void *b_) diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index 78ccf4b33f..4a8a78769b 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -343,6 +343,18 @@ test_expect_success 'maintenance.incremental-repack.auto' ' test_subcommand git multi-pack-index write --no-progress after && + test_must_be_empty after && + test_subcommand git pack-refs --all --prune err && test_i18ngrep "at most one" err From acc1c4d5d4c80c4fd93aaa151caa5593656600ae Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 9 Feb 2021 13:42:29 +0000 Subject: [PATCH 2/2] maintenance: incremental strategy runs pack-refs weekly When the 'maintenance.strategy' config option is set to 'incremental', a default maintenance schedule is enabled. Add the 'pack-refs' task to that strategy at the weekly cadence. Signed-off-by: Derrick Stolee Reviewed-by: Taylor Blau Signed-off-by: Junio C Hamano --- Documentation/config/maintenance.txt | 5 +++-- builtin/gc.c | 2 ++ t/t7900-maintenance.sh | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/Documentation/config/maintenance.txt b/Documentation/config/maintenance.txt index a5ead09e4b..18f0562131 100644 --- a/Documentation/config/maintenance.txt +++ b/Documentation/config/maintenance.txt @@ -15,8 +15,9 @@ maintenance.strategy:: * `none`: This default setting implies no task are run at any schedule. * `incremental`: This setting optimizes for performing small maintenance activities that do not delete any data. This does not schedule the `gc` - task, but runs the `prefetch` and `commit-graph` tasks hourly and the - `loose-objects` and `incremental-repack` tasks daily. + task, but runs the `prefetch` and `commit-graph` tasks hourly, the + `loose-objects` and `incremental-repack` tasks daily, and the `pack-refs` + task weekly. maintenance..enabled:: This boolean config option controls whether the maintenance task diff --git a/builtin/gc.c b/builtin/gc.c index 41bec4f177..6db9cb39e6 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -1352,6 +1352,8 @@ static void initialize_maintenance_strategy(void) tasks[TASK_INCREMENTAL_REPACK].schedule = SCHEDULE_DAILY; tasks[TASK_LOOSE_OBJECTS].enabled = 1; tasks[TASK_LOOSE_OBJECTS].schedule = SCHEDULE_DAILY; + tasks[TASK_PACK_REFS].enabled = 1; + tasks[TASK_PACK_REFS].schedule = SCHEDULE_WEEKLY; } } diff --git a/t/t7900-maintenance.sh b/t/t7900-maintenance.sh index 4a8a78769b..286b18db3c 100755 --- a/t/t7900-maintenance.sh +++ b/t/t7900-maintenance.sh @@ -408,18 +408,32 @@ test_expect_success 'maintenance.strategy inheritance' ' git maintenance run --schedule=hourly --quiet && GIT_TRACE2_EVENT="$(pwd)/incremental-daily.txt" \ git maintenance run --schedule=daily --quiet && + GIT_TRACE2_EVENT="$(pwd)/incremental-weekly.txt" \ + git maintenance run --schedule=weekly --quiet && test_subcommand git commit-graph write --split --reachable \ --no-progress