From 7afc0b03a2307aa02b7caff2746664e886f38d9a Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Fri, 30 Jul 2021 11:47:36 +0000 Subject: [PATCH 1/9] merge-ort: rename str{map,intmap,set}_func() In order to make it clearer that these three variables holding a function refer to functions that will clear the strmap/strintmap/strset, rename them to str{map,intmap,set}_clear_func(). Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- merge-ort.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/merge-ort.c b/merge-ort.c index e75b524153..401a40247a 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -519,11 +519,11 @@ static void clear_or_reinit_internal_opts(struct merge_options_internal *opti, { struct rename_info *renames = &opti->renames; int i; - void (*strmap_func)(struct strmap *, int) = + void (*strmap_clear_func)(struct strmap *, int) = reinitialize ? strmap_partial_clear : strmap_clear; - void (*strintmap_func)(struct strintmap *) = + void (*strintmap_clear_func)(struct strintmap *) = reinitialize ? strintmap_partial_clear : strintmap_clear; - void (*strset_func)(struct strset *) = + void (*strset_clear_func)(struct strset *) = reinitialize ? strset_partial_clear : strset_clear; /* @@ -534,14 +534,14 @@ static void clear_or_reinit_internal_opts(struct merge_options_internal *opti, * to deallocate them. */ free_strmap_strings(&opti->paths); - strmap_func(&opti->paths, 1); + strmap_clear_func(&opti->paths, 1); /* * All keys and values in opti->conflicted are a subset of those in * opti->paths. We don't want to deallocate anything twice, so we * don't free the keys and we pass 0 for free_values. */ - strmap_func(&opti->conflicted, 0); + strmap_clear_func(&opti->conflicted, 0); /* * opti->paths_to_free is similar to opti->paths; we created it with @@ -559,24 +559,24 @@ static void clear_or_reinit_internal_opts(struct merge_options_internal *opti, /* Free memory used by various renames maps */ for (i = MERGE_SIDE1; i <= MERGE_SIDE2; ++i) { - strintmap_func(&renames->dirs_removed[i]); - strmap_func(&renames->dir_renames[i], 0); - strintmap_func(&renames->relevant_sources[i]); + strintmap_clear_func(&renames->dirs_removed[i]); + strmap_clear_func(&renames->dir_renames[i], 0); + strintmap_clear_func(&renames->relevant_sources[i]); if (!reinitialize) assert(renames->cached_pairs_valid_side == 0); if (i != renames->cached_pairs_valid_side && -1 != renames->cached_pairs_valid_side) { - strset_func(&renames->cached_target_names[i]); - strmap_func(&renames->cached_pairs[i], 1); - strset_func(&renames->cached_irrelevant[i]); + strset_clear_func(&renames->cached_target_names[i]); + strmap_clear_func(&renames->cached_pairs[i], 1); + strset_clear_func(&renames->cached_irrelevant[i]); partial_clear_dir_rename_count(&renames->dir_rename_count[i]); if (!reinitialize) strmap_clear(&renames->dir_rename_count[i], 1); } } for (i = MERGE_SIDE1; i <= MERGE_SIDE2; ++i) { - strintmap_func(&renames->deferred[i].possible_trivial_merges); - strset_func(&renames->deferred[i].target_dirs); + strintmap_clear_func(&renames->deferred[i].possible_trivial_merges); + strset_clear_func(&renames->deferred[i].target_dirs); renames->deferred[i].trivial_merges_okay = 1; /* 1 == maybe */ } renames->cached_pairs_valid_side = 0; From fa0e936fbb683dbce70b797ae8a5be6b88f48d88 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Fri, 30 Jul 2021 11:47:37 +0000 Subject: [PATCH 2/9] diffcore-rename: use a mem_pool for exact rename detection's hashmap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exact rename detection, via insert_file_table(), uses a hashmap to store files by oid. Use a mem_pool for the hashmap entries so these can all be allocated and deallocated together. For the testcases mentioned in commit 557ac0350d ("merge-ort: begin performance work; instrument with trace2_region_* calls", 2020-10-28), this change improves the performance as follows: Before After no-renames: 204.2 ms ± 3.0 ms 202.5 ms ± 3.2 ms mega-renames: 1.076 s ± 0.015 s 1.072 s ± 0.012 s just-one-mega: 364.1 ms ± 7.0 ms 357.3 ms ± 3.9 ms Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- diffcore-rename.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/diffcore-rename.c b/diffcore-rename.c index 4ef0459cfb..73d884099e 100644 --- a/diffcore-rename.c +++ b/diffcore-rename.c @@ -317,10 +317,11 @@ static int find_identical_files(struct hashmap *srcs, } static void insert_file_table(struct repository *r, + struct mem_pool *pool, struct hashmap *table, int index, struct diff_filespec *filespec) { - struct file_similarity *entry = xmalloc(sizeof(*entry)); + struct file_similarity *entry = mem_pool_alloc(pool, sizeof(*entry)); entry->index = index; entry->filespec = filespec; @@ -336,7 +337,8 @@ static void insert_file_table(struct repository *r, * and then during the second round we try to match * cache-dirty entries as well. */ -static int find_exact_renames(struct diff_options *options) +static int find_exact_renames(struct diff_options *options, + struct mem_pool *pool) { int i, renames = 0; struct hashmap file_table; @@ -346,7 +348,7 @@ static int find_exact_renames(struct diff_options *options) */ hashmap_init(&file_table, NULL, NULL, rename_src_nr); for (i = rename_src_nr-1; i >= 0; i--) - insert_file_table(options->repo, + insert_file_table(options->repo, pool, &file_table, i, rename_src[i].p->one); @@ -354,8 +356,8 @@ static int find_exact_renames(struct diff_options *options) for (i = 0; i < rename_dst_nr; i++) renames += find_identical_files(&file_table, i, options); - /* Free the hash data structure and entries */ - hashmap_clear_and_free(&file_table, struct file_similarity, entry); + /* Free the hash data structure (entries will be freed with the pool) */ + hashmap_clear(&file_table); return renames; } @@ -1341,6 +1343,7 @@ void diffcore_rename_extended(struct diff_options *options, int num_destinations, dst_cnt; int num_sources, want_copies; struct progress *progress = NULL; + struct mem_pool local_pool; struct dir_rename_info info; struct diff_populate_filespec_options dpf_options = { .check_binary = 0, @@ -1409,11 +1412,18 @@ void diffcore_rename_extended(struct diff_options *options, goto cleanup; /* nothing to do */ trace2_region_enter("diff", "exact renames", options->repo); + mem_pool_init(&local_pool, 32*1024); /* * We really want to cull the candidates list early * with cheap tests in order to avoid doing deltas. */ - rename_count = find_exact_renames(options); + rename_count = find_exact_renames(options, &local_pool); + /* + * Discard local_pool immediately instead of at "cleanup:" in order + * to reduce maximum memory usage; inexact rename detection uses up + * a fair amount of memory, and mem_pools can too. + */ + mem_pool_discard(&local_pool, 0); trace2_region_leave("diff", "exact renames", options->repo); /* Did we only want exact renames? */ From cdf2241c717b05845ff9e25b87be6ca7559e82eb Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Fri, 30 Jul 2021 11:47:38 +0000 Subject: [PATCH 3/9] merge-ort: add pool_alloc, pool_calloc, and pool_strndup wrappers Make the code more flexible so that it can handle both being run with or without a memory pool by adding utility functions which will either call xmalloc, xcalloc, xstrndup or mem_pool_alloc, mem_pool_calloc, mem_pool_strndup depending on whether we have a non-NULL memory pool. A subsequent commit will make use of these. (We will actually be dropping these functions soon and just assuming we always have a memory pool, but the flexibility was very useful during development of merge-ort so I want to be able to restore it if needed.) Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- merge-ort.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/merge-ort.c b/merge-ort.c index 401a40247a..63f67246d3 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -664,6 +664,30 @@ static void path_msg(struct merge_options *opt, strbuf_addch(sb, '\n'); } +MAYBE_UNUSED +static void *pool_calloc(struct mem_pool *pool, size_t count, size_t size) +{ + if (!pool) + return xcalloc(count, size); + return mem_pool_calloc(pool, count, size); +} + +MAYBE_UNUSED +static void *pool_alloc(struct mem_pool *pool, size_t size) +{ + if (!pool) + return xmalloc(size); + return mem_pool_alloc(pool, size); +} + +MAYBE_UNUSED +static void *pool_strndup(struct mem_pool *pool, const char *str, size_t len) +{ + if (!pool) + return xstrndup(str, len); + return mem_pool_strndup(pool, str, len); +} + /* add a string to a strbuf, but converting "/" to "_" */ static void add_flattened_path(struct strbuf *out, const char *s) { From 4137c54b902faa640c485dc6d20eaac946e503a5 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Fri, 30 Jul 2021 11:47:39 +0000 Subject: [PATCH 4/9] merge-ort: set up a memory pool merge-ort has a lot of data structures, and they all tend to be freed together in clear_or_reinit_internal_opts(). Set up a memory pool to allow us to make these allocations and deallocations faster. Future commits will adjust various callers to make use of this memory pool. Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- merge-ort.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/merge-ort.c b/merge-ort.c index 63f67246d3..3f42543626 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -37,6 +37,8 @@ #include "unpack-trees.h" #include "xdiff-interface.h" +#define USE_MEMORY_POOL 1 /* faster, but obscures memory leak hunting */ + /* * We have many arrays of size 3. Whenever we have such an array, the * indices refer to one of the sides of the three-way merge. This is so @@ -339,6 +341,17 @@ struct merge_options_internal { */ struct strmap conflicted; + /* + * pool: memory pool for fast allocation/deallocation + * + * We allocate room for lots of filenames and auxiliary data + * structures in merge_options_internal, and it tends to all be + * freed together too. Using a memory pool for these provides a + * nice speedup. + */ + struct mem_pool internal_pool; + struct mem_pool *pool; /* NULL, or pointer to internal_pool */ + /* * paths_to_free: additional list of strings to free * @@ -603,6 +616,12 @@ static void clear_or_reinit_internal_opts(struct merge_options_internal *opti, strmap_clear(&opti->output, 0); } +#if USE_MEMORY_POOL + mem_pool_discard(&opti->internal_pool, 0); + if (!reinitialize) + opti->pool = NULL; +#endif + /* Clean out callback_data as well. */ FREE_AND_NULL(renames->callback_data); renames->callback_data_nr = renames->callback_data_alloc = 0; @@ -4381,6 +4400,12 @@ static void merge_start(struct merge_options *opt, struct merge_result *result) /* Initialization of various renames fields */ renames = &opt->priv->renames; +#if USE_MEMORY_POOL + mem_pool_init(&opt->priv->internal_pool, 0); + opt->priv->pool = &opt->priv->internal_pool; +#else + opt->priv->pool = NULL; +#endif for (i = MERGE_SIDE1; i <= MERGE_SIDE2; i++) { strintmap_init_with_options(&renames->dirs_removed[i], NOT_RELEVANT, NULL, 0); From 6697ee01b5d31df5c83ace9eabb6285cf42ff172 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Fri, 30 Jul 2021 11:47:40 +0000 Subject: [PATCH 5/9] merge-ort: switch our strmaps over to using memory pools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For all the strmaps (including strintmaps and strsets) whose memory is unconditionally freed as part of clear_or_reinit_internal_opts(), switch them over to using our new memory pool. For the testcases mentioned in commit 557ac0350d ("merge-ort: begin performance work; instrument with trace2_region_* calls", 2020-10-28), this change improves the performance as follows: Before After no-renames: 202.5 ms ± 3.2 ms 198.1 ms ± 2.6 ms mega-renames: 1.072 s ± 0.012 s 715.8 ms ± 4.0 ms just-one-mega: 357.3 ms ± 3.9 ms 276.8 ms ± 4.2 ms Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- merge-ort.c | 125 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 75 insertions(+), 50 deletions(-) diff --git a/merge-ort.c b/merge-ort.c index 3f42543626..99c7569085 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -539,15 +539,19 @@ static void clear_or_reinit_internal_opts(struct merge_options_internal *opti, void (*strset_clear_func)(struct strset *) = reinitialize ? strset_partial_clear : strset_clear; - /* - * We marked opti->paths with strdup_strings = 0, so that we - * wouldn't have to make another copy of the fullpath created by - * make_traverse_path from setup_path_info(). But, now that we've - * used it and have no other references to these strings, it is time - * to deallocate them. - */ - free_strmap_strings(&opti->paths); - strmap_clear_func(&opti->paths, 1); + if (opti->pool) + strmap_clear_func(&opti->paths, 0); + else { + /* + * We marked opti->paths with strdup_strings = 0, so that + * we wouldn't have to make another copy of the fullpath + * created by make_traverse_path from setup_path_info(). + * But, now that we've used it and have no other references + * to these strings, it is time to deallocate them. + */ + free_strmap_strings(&opti->paths); + strmap_clear_func(&opti->paths, 1); + } /* * All keys and values in opti->conflicted are a subset of those in @@ -556,16 +560,19 @@ static void clear_or_reinit_internal_opts(struct merge_options_internal *opti, */ strmap_clear_func(&opti->conflicted, 0); - /* - * opti->paths_to_free is similar to opti->paths; we created it with - * strdup_strings = 0 to avoid making _another_ copy of the fullpath - * but now that we've used it and have no other references to these - * strings, it is time to deallocate them. We do so by temporarily - * setting strdup_strings to 1. - */ - opti->paths_to_free.strdup_strings = 1; - string_list_clear(&opti->paths_to_free, 0); - opti->paths_to_free.strdup_strings = 0; + if (!opti->pool) { + /* + * opti->paths_to_free is similar to opti->paths; we + * created it with strdup_strings = 0 to avoid making + * _another_ copy of the fullpath but now that we've used + * it and have no other references to these strings, it is + * time to deallocate them. We do so by temporarily + * setting strdup_strings to 1. + */ + opti->paths_to_free.strdup_strings = 1; + string_list_clear(&opti->paths_to_free, 0); + opti->paths_to_free.strdup_strings = 0; + } if (opti->attr_index.cache_nr) /* true iff opt->renormalize */ discard_index(&opti->attr_index); @@ -683,7 +690,6 @@ static void path_msg(struct merge_options *opt, strbuf_addch(sb, '\n'); } -MAYBE_UNUSED static void *pool_calloc(struct mem_pool *pool, size_t count, size_t size) { if (!pool) @@ -691,7 +697,6 @@ static void *pool_calloc(struct mem_pool *pool, size_t count, size_t size) return mem_pool_calloc(pool, count, size); } -MAYBE_UNUSED static void *pool_alloc(struct mem_pool *pool, size_t size) { if (!pool) @@ -699,7 +704,6 @@ static void *pool_alloc(struct mem_pool *pool, size_t size) return mem_pool_alloc(pool, size); } -MAYBE_UNUSED static void *pool_strndup(struct mem_pool *pool, const char *str, size_t len) { if (!pool) @@ -835,8 +839,9 @@ static void setup_path_info(struct merge_options *opt, assert(!df_conflict || !resolved); /* df_conflict implies !resolved */ assert(resolved == (merged_version != NULL)); - mi = xcalloc(1, resolved ? sizeof(struct merged_info) : - sizeof(struct conflict_info)); + mi = pool_calloc(opt->priv->pool, 1, + resolved ? sizeof(struct merged_info) : + sizeof(struct conflict_info)); mi->directory_name = current_dir_name; mi->basename_offset = current_dir_name_len; mi->clean = !!resolved; @@ -1128,7 +1133,7 @@ static int collect_merge_info_callback(int n, len = traverse_path_len(info, p->pathlen); /* +1 in both of the following lines to include the NUL byte */ - fullpath = xmalloc(len + 1); + fullpath = pool_alloc(opt->priv->pool, len + 1); make_traverse_path(fullpath, len + 1, info, p->path, p->pathlen); /* @@ -1383,7 +1388,7 @@ static int handle_deferred_entries(struct merge_options *opt, copy = renames->deferred[side].possible_trivial_merges; strintmap_init_with_options(&renames->deferred[side].possible_trivial_merges, 0, - NULL, + opt->priv->pool, 0); strintmap_for_each_entry(©, &iter, entry) { const char *path = entry->key; @@ -2335,12 +2340,21 @@ static void apply_directory_rename_modifications(struct merge_options *opt, VERIFY_CI(ci); /* Find parent directories missing from opt->priv->paths */ - cur_path = new_path; + if (opt->priv->pool) { + cur_path = mem_pool_strdup(opt->priv->pool, new_path); + free((char*)new_path); + new_path = (char *)cur_path; + } else { + cur_path = new_path; + } + while (1) { /* Find the parent directory of cur_path */ char *last_slash = strrchr(cur_path, '/'); if (last_slash) { - parent_name = xstrndup(cur_path, last_slash - cur_path); + parent_name = pool_strndup(opt->priv->pool, + cur_path, + last_slash - cur_path); } else { parent_name = opt->priv->toplevel_dir; break; @@ -2349,7 +2363,8 @@ static void apply_directory_rename_modifications(struct merge_options *opt, /* Look it up in opt->priv->paths */ entry = strmap_get_entry(&opt->priv->paths, parent_name); if (entry) { - free((char*)parent_name); + if (!opt->priv->pool) + free((char*)parent_name); parent_name = entry->key; /* reuse known pointer */ break; } @@ -2376,12 +2391,15 @@ static void apply_directory_rename_modifications(struct merge_options *opt, parent_name = cur_dir; } - /* - * We are removing old_path from opt->priv->paths. old_path also will - * eventually need to be freed, but it may still be used by e.g. - * ci->pathnames. So, store it in another string-list for now. - */ - string_list_append(&opt->priv->paths_to_free, old_path); + if (!opt->priv->pool) { + /* + * We are removing old_path from opt->priv->paths. + * old_path also will eventually need to be freed, but it + * may still be used by e.g. ci->pathnames. So, store it + * in another string-list for now. + */ + string_list_append(&opt->priv->paths_to_free, old_path); + } assert(ci->filemask == 2 || ci->filemask == 4); assert(ci->dirmask == 0); @@ -2416,7 +2434,8 @@ static void apply_directory_rename_modifications(struct merge_options *opt, new_ci->stages[index].mode = ci->stages[index].mode; oidcpy(&new_ci->stages[index].oid, &ci->stages[index].oid); - free(ci); + if (!opt->priv->pool) + free(ci); ci = new_ci; } @@ -3623,7 +3642,8 @@ static void process_entry(struct merge_options *opt, * the directory to remain here, so we need to move this * path to some new location. */ - CALLOC_ARRAY(new_ci, 1); + new_ci = pool_calloc(opt->priv->pool, 1, sizeof(*new_ci)); + /* We don't really want new_ci->merged.result copied, but it'll * be overwritten below so it doesn't matter. We also don't * want any directory mode/oid values copied, but we'll zero @@ -3715,7 +3735,7 @@ static void process_entry(struct merge_options *opt, const char *a_path = NULL, *b_path = NULL; int rename_a = 0, rename_b = 0; - new_ci = xmalloc(sizeof(*new_ci)); + new_ci = pool_alloc(opt->priv->pool, sizeof(*new_ci)); if (S_ISREG(a_mode)) rename_a = 1; @@ -3788,12 +3808,14 @@ static void process_entry(struct merge_options *opt, strmap_remove(&opt->priv->paths, path, 0); /* * We removed path from opt->priv->paths. path - * will also eventually need to be freed, but - * it may still be used by e.g. ci->pathnames. - * So, store it in another string-list for now. + * will also eventually need to be freed if not + * part of a memory pool...but it may still be + * used by e.g. ci->pathnames. So, store it in + * another string-list for now in that case. */ - string_list_append(&opt->priv->paths_to_free, - path); + if (!opt->priv->pool) + string_list_append(&opt->priv->paths_to_free, + path); } /* @@ -4335,6 +4357,7 @@ static void merge_start(struct merge_options *opt, struct merge_result *result) { struct rename_info *renames; int i; + struct mem_pool *pool = NULL; /* Sanity checks on opt */ trace2_region_enter("merge", "sanity checks", opt->repo); @@ -4406,9 +4429,10 @@ static void merge_start(struct merge_options *opt, struct merge_result *result) #else opt->priv->pool = NULL; #endif + pool = opt->priv->pool; for (i = MERGE_SIDE1; i <= MERGE_SIDE2; i++) { strintmap_init_with_options(&renames->dirs_removed[i], - NOT_RELEVANT, NULL, 0); + NOT_RELEVANT, pool, 0); strmap_init_with_options(&renames->dir_rename_count[i], NULL, 1); strmap_init_with_options(&renames->dir_renames[i], @@ -4422,7 +4446,7 @@ static void merge_start(struct merge_options *opt, struct merge_result *result) */ strintmap_init_with_options(&renames->relevant_sources[i], -1 /* explicitly invalid */, - NULL, 0); + pool, 0); strmap_init_with_options(&renames->cached_pairs[i], NULL, 1); strset_init_with_options(&renames->cached_irrelevant[i], @@ -4432,9 +4456,9 @@ static void merge_start(struct merge_options *opt, struct merge_result *result) } for (i = MERGE_SIDE1; i <= MERGE_SIDE2; i++) { strintmap_init_with_options(&renames->deferred[i].possible_trivial_merges, - 0, NULL, 0); + 0, pool, 0); strset_init_with_options(&renames->deferred[i].target_dirs, - NULL, 1); + pool, 1); renames->deferred[i].trivial_merges_okay = 1; /* 1 == maybe */ } @@ -4447,9 +4471,10 @@ static void merge_start(struct merge_options *opt, struct merge_result *result) * In contrast, conflicted just has a subset of keys from paths, so * we don't want to free those (it'd be a duplicate free). */ - strmap_init_with_options(&opt->priv->paths, NULL, 0); - strmap_init_with_options(&opt->priv->conflicted, NULL, 0); - string_list_init_nodup(&opt->priv->paths_to_free); + strmap_init_with_options(&opt->priv->paths, pool, 0); + strmap_init_with_options(&opt->priv->conflicted, pool, 0); + if (!opt->priv->pool) + string_list_init_nodup(&opt->priv->paths_to_free); /* * keys & strbufs in output will sometimes need to outlive "paths", From a8791ef6492bf702ba70352009cbd28fb581c6e2 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Fri, 30 Jul 2021 11:47:41 +0000 Subject: [PATCH 6/9] diffcore-rename, merge-ort: add wrapper functions for filepair alloc/dealloc We want to be able to allocate filespecs and filepairs using a mem_pool. However, filespec data will still remain outside the pool (perhaps in the future we could plumb the pool through the various diff APIs to allocate the filespec data too, but for now we are limiting the scope). Add some extra functions to allocate these appropriately based on the non-NULL-ness of opt->priv->pool, as well as some extra functions to handle correctly deallocating the relevant parts of them. A future commit will make use of these new functions. Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- diffcore-rename.c | 41 +++++++++++++++++++++++++++++++++++++++++ diffcore.h | 2 ++ merge-ort.c | 42 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) diff --git a/diffcore-rename.c b/diffcore-rename.c index 73d884099e..5bc559f79e 100644 --- a/diffcore-rename.c +++ b/diffcore-rename.c @@ -1328,6 +1328,47 @@ static void handle_early_known_dir_renames(struct dir_rename_info *info, rename_src_nr = new_num_src; } +static void free_filespec_data(struct diff_filespec *spec) +{ + if (!--spec->count) + diff_free_filespec_data(spec); +} + +MAYBE_UNUSED +static void pool_free_filespec(struct mem_pool *pool, + struct diff_filespec *spec) +{ + if (!pool) { + free_filespec(spec); + return; + } + + /* + * Similar to free_filespec(), but only frees the data. The spec + * itself was allocated in the pool and should not be individually + * freed. + */ + free_filespec_data(spec); +} + +MAYBE_UNUSED +void pool_diff_free_filepair(struct mem_pool *pool, + struct diff_filepair *p) +{ + if (!pool) { + diff_free_filepair(p); + return; + } + + /* + * Similar to diff_free_filepair() but only frees the data from the + * filespecs; not the filespecs or the filepair which were + * allocated from the pool. + */ + free_filespec_data(p->one); + free_filespec_data(p->two); +} + void diffcore_rename_extended(struct diff_options *options, struct strintmap *relevant_sources, struct strintmap *dirs_removed, diff --git a/diffcore.h b/diffcore.h index 533b30e21e..b58ee6b193 100644 --- a/diffcore.h +++ b/diffcore.h @@ -127,6 +127,8 @@ struct diff_filepair { #define DIFF_PAIR_MODE_CHANGED(p) ((p)->one->mode != (p)->two->mode) void diff_free_filepair(struct diff_filepair *); +void pool_diff_free_filepair(struct mem_pool *pool, + struct diff_filepair *p); int diff_unmodified_pair(struct diff_filepair *); diff --git a/merge-ort.c b/merge-ort.c index 99c7569085..e79830f918 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -690,6 +690,48 @@ static void path_msg(struct merge_options *opt, strbuf_addch(sb, '\n'); } +MAYBE_UNUSED +static struct diff_filespec *pool_alloc_filespec(struct mem_pool *pool, + const char *path) +{ + struct diff_filespec *spec; + size_t len; + + if (!pool) + return alloc_filespec(path); + + /* Same code as alloc_filespec, except allocate from pool */ + len = strlen(path); + + spec = mem_pool_calloc(pool, 1, st_add3(sizeof(*spec), len, 1)); + memcpy(spec+1, path, len); + spec->path = (void*)(spec+1); + + spec->count = 1; + spec->is_binary = -1; + return spec; +} + +MAYBE_UNUSED +static struct diff_filepair *pool_diff_queue(struct mem_pool *pool, + struct diff_queue_struct *queue, + struct diff_filespec *one, + struct diff_filespec *two) +{ + struct diff_filepair *dp; + + if (!pool) + return diff_queue(queue, one, two); + + /* Same code as diff_queue, except allocate from pool */ + dp = mem_pool_calloc(pool, 1, sizeof(*dp)); + dp->one = one; + dp->two = two; + if (queue) + diff_q(queue, dp); + return dp; +} + static void *pool_calloc(struct mem_pool *pool, size_t count, size_t size) { if (!pool) From f239fff4c183b231134115bc3b38b4f8e3982d3e Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Fri, 30 Jul 2021 11:47:42 +0000 Subject: [PATCH 7/9] merge-ort: store filepairs and filespecs in our mem_pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For the testcases mentioned in commit 557ac0350d ("merge-ort: begin performance work; instrument with trace2_region_* calls", 2020-10-28), this change improves the performance as follows: Before After no-renames: 198.1 ms ± 2.6 ms 198.5 ms ± 3.4 ms mega-renames: 715.8 ms ± 4.0 ms 679.1 ms ± 5.6 ms just-one-mega: 276.8 ms ± 4.2 ms 271.9 ms ± 2.8 ms Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- diffcore-rename.c | 9 ++++----- diffcore.h | 1 + merge-ort.c | 26 ++++++++++++++------------ 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/diffcore-rename.c b/diffcore-rename.c index 5bc559f79e..7e6b3e1b14 100644 --- a/diffcore-rename.c +++ b/diffcore-rename.c @@ -1334,7 +1334,6 @@ static void free_filespec_data(struct diff_filespec *spec) diff_free_filespec_data(spec); } -MAYBE_UNUSED static void pool_free_filespec(struct mem_pool *pool, struct diff_filespec *spec) { @@ -1351,7 +1350,6 @@ static void pool_free_filespec(struct mem_pool *pool, free_filespec_data(spec); } -MAYBE_UNUSED void pool_diff_free_filepair(struct mem_pool *pool, struct diff_filepair *p) { @@ -1370,6 +1368,7 @@ void pool_diff_free_filepair(struct mem_pool *pool, } void diffcore_rename_extended(struct diff_options *options, + struct mem_pool *pool, struct strintmap *relevant_sources, struct strintmap *dirs_removed, struct strmap *dir_rename_count, @@ -1683,7 +1682,7 @@ void diffcore_rename_extended(struct diff_options *options, pair_to_free = p; if (pair_to_free) - diff_free_filepair(pair_to_free); + pool_diff_free_filepair(pool, pair_to_free); } diff_debug_queue("done copying original", &outq); @@ -1693,7 +1692,7 @@ void diffcore_rename_extended(struct diff_options *options, for (i = 0; i < rename_dst_nr; i++) if (rename_dst[i].filespec_to_free) - free_filespec(rename_dst[i].filespec_to_free); + pool_free_filespec(pool, rename_dst[i].filespec_to_free); cleanup_dir_rename_info(&info, dirs_removed, dir_rename_count != NULL); FREE_AND_NULL(rename_dst); @@ -1710,5 +1709,5 @@ void diffcore_rename_extended(struct diff_options *options, void diffcore_rename(struct diff_options *options) { - diffcore_rename_extended(options, NULL, NULL, NULL, NULL); + diffcore_rename_extended(options, NULL, NULL, NULL, NULL, NULL); } diff --git a/diffcore.h b/diffcore.h index b58ee6b193..badc2261c2 100644 --- a/diffcore.h +++ b/diffcore.h @@ -181,6 +181,7 @@ void partial_clear_dir_rename_count(struct strmap *dir_rename_count); void diffcore_break(struct repository *, int); void diffcore_rename(struct diff_options *); void diffcore_rename_extended(struct diff_options *options, + struct mem_pool *pool, struct strintmap *relevant_sources, struct strintmap *dirs_removed, struct strmap *dir_rename_count, diff --git a/merge-ort.c b/merge-ort.c index e79830f918..f4f0a3d57f 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -690,7 +690,6 @@ static void path_msg(struct merge_options *opt, strbuf_addch(sb, '\n'); } -MAYBE_UNUSED static struct diff_filespec *pool_alloc_filespec(struct mem_pool *pool, const char *path) { @@ -712,7 +711,6 @@ static struct diff_filespec *pool_alloc_filespec(struct mem_pool *pool, return spec; } -MAYBE_UNUSED static struct diff_filepair *pool_diff_queue(struct mem_pool *pool, struct diff_queue_struct *queue, struct diff_filespec *one, @@ -930,6 +928,7 @@ static void add_pair(struct merge_options *opt, unsigned dir_rename_mask) { struct diff_filespec *one, *two; + struct mem_pool *pool = opt->priv->pool; struct rename_info *renames = &opt->priv->renames; int names_idx = is_add ? side : 0; @@ -980,11 +979,11 @@ static void add_pair(struct merge_options *opt, return; } - one = alloc_filespec(pathname); - two = alloc_filespec(pathname); + one = pool_alloc_filespec(pool, pathname); + two = pool_alloc_filespec(pool, pathname); fill_filespec(is_add ? two : one, &names[names_idx].oid, 1, names[names_idx].mode); - diff_queue(&renames->pairs[side], one, two); + pool_diff_queue(pool, &renames->pairs[side], one, two); } static void collect_rename_info(struct merge_options *opt, @@ -2893,6 +2892,7 @@ static void use_cached_pairs(struct merge_options *opt, { struct hashmap_iter iter; struct strmap_entry *entry; + struct mem_pool *pool = opt->priv->pool; /* * Add to side_pairs all entries from renames->cached_pairs[side_index]. @@ -2906,9 +2906,9 @@ static void use_cached_pairs(struct merge_options *opt, new_name = old_name; /* We don't care about oid/mode, only filenames and status */ - one = alloc_filespec(old_name); - two = alloc_filespec(new_name); - diff_queue(pairs, one, two); + one = pool_alloc_filespec(pool, old_name); + two = pool_alloc_filespec(pool, new_name); + pool_diff_queue(pool, pairs, one, two); pairs->queue[pairs->nr-1]->status = entry->value ? 'R' : 'D'; } } @@ -3016,6 +3016,7 @@ static int detect_regular_renames(struct merge_options *opt, diff_queued_diff = renames->pairs[side_index]; trace2_region_enter("diff", "diffcore_rename", opt->repo); diffcore_rename_extended(&diff_opts, + opt->priv->pool, &renames->relevant_sources[side_index], &renames->dirs_removed[side_index], &renames->dir_rename_count[side_index], @@ -3066,7 +3067,7 @@ static int collect_renames(struct merge_options *opt, if (p->status != 'A' && p->status != 'R') { possibly_cache_new_pair(renames, p, side_index, NULL); - diff_free_filepair(p); + pool_diff_free_filepair(opt->priv->pool, p); continue; } @@ -3079,7 +3080,7 @@ static int collect_renames(struct merge_options *opt, possibly_cache_new_pair(renames, p, side_index, new_path); if (p->status != 'R' && !new_path) { - diff_free_filepair(p); + pool_diff_free_filepair(opt->priv->pool, p); continue; } @@ -3197,7 +3198,7 @@ static int detect_and_process_renames(struct merge_options *opt, side_pairs = &renames->pairs[s]; for (i = 0; i < side_pairs->nr; ++i) { struct diff_filepair *p = side_pairs->queue[i]; - diff_free_filepair(p); + pool_diff_free_filepair(opt->priv->pool, p); } } @@ -3210,7 +3211,8 @@ static int detect_and_process_renames(struct merge_options *opt, if (combined.nr) { int i; for (i = 0; i < combined.nr; i++) - diff_free_filepair(combined.queue[i]); + pool_diff_free_filepair(opt->priv->pool, + combined.queue[i]); free(combined.queue); } From 092e5115d1312c0b30348404d3f5b2bb02909342 Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Fri, 30 Jul 2021 11:47:43 +0000 Subject: [PATCH 8/9] merge-ort: reuse path strings in pool_alloc_filespec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pool_alloc_filespec() was written so that the code when pool != NULL mimicked the code from alloc_filespec(), which including allocating enough extra space for the path and then copying it. However, the path passed to pool_alloc_filespec() is always going to already be in the same memory pool, so we may as well reuse it instead of copying it. For the testcases mentioned in commit 557ac0350d ("merge-ort: begin performance work; instrument with trace2_region_* calls", 2020-10-28), this change improves the performance as follows: Before After no-renames: 198.5 ms ± 3.4 ms 198.3 ms ± 2.9 ms mega-renames: 679.1 ms ± 5.6 ms 661.8 ms ± 5.9 ms just-one-mega: 271.9 ms ± 2.8 ms 264.6 ms ± 2.5 ms Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- merge-ort.c | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/merge-ort.c b/merge-ort.c index f4f0a3d57f..86ab8f6012 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -694,17 +694,13 @@ static struct diff_filespec *pool_alloc_filespec(struct mem_pool *pool, const char *path) { struct diff_filespec *spec; - size_t len; if (!pool) return alloc_filespec(path); - /* Same code as alloc_filespec, except allocate from pool */ - len = strlen(path); - - spec = mem_pool_calloc(pool, 1, st_add3(sizeof(*spec), len, 1)); - memcpy(spec+1, path, len); - spec->path = (void*)(spec+1); + /* Similar to alloc_filespec, but allocate from pool and reuse path */ + spec = mem_pool_calloc(pool, 1, sizeof(*spec)); + spec->path = (char*)path; /* spec won't modify it */ spec->count = 1; spec->is_binary = -1; @@ -2904,6 +2900,25 @@ static void use_cached_pairs(struct merge_options *opt, const char *new_name = entry->value; if (!new_name) new_name = old_name; + if (pool) { + /* + * cached_pairs has _copies* of old_name and new_name, + * because it has to persist across merges. When + * pool != NULL + * pool_alloc_filespec() will just re-use the existing + * filenames, which will also get re-used by + * opt->priv->paths if they become renames, and then + * get freed at the end of the merge, leaving the copy + * in cached_pairs dangling. Avoid this by making a + * copy here. + * + * When pool == NULL, pool_alloc_filespec() calls + * alloc_filespec(), which makes a copy; we don't want + * to add another. + */ + old_name = mem_pool_strdup(pool, old_name); + new_name = mem_pool_strdup(pool, new_name); + } /* We don't care about oid/mode, only filenames and status */ one = pool_alloc_filespec(pool, old_name); From 62a15162fe60f11c515bec116c9a1f58173a032f Mon Sep 17 00:00:00 2001 From: Elijah Newren Date: Sat, 31 Jul 2021 17:27:38 +0000 Subject: [PATCH 9/9] merge-ort: remove compile-time ability to turn off usage of memory pools Simplify code maintenance by removing the ability to toggle between usage of memory pools and direct allocations. This allows us to also remove paths_to_free since it was solely about bookkeeping to make sure we freed the necessary paths, and allows us to remove some auxiliary functions. Suggested-by: Jeff King Signed-off-by: Elijah Newren Signed-off-by: Junio C Hamano --- merge-ort.c | 209 ++++++++++++---------------------------------------- 1 file changed, 47 insertions(+), 162 deletions(-) diff --git a/merge-ort.c b/merge-ort.c index 86ab8f6012..88ade50f4e 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -37,8 +37,6 @@ #include "unpack-trees.h" #include "xdiff-interface.h" -#define USE_MEMORY_POOL 1 /* faster, but obscures memory leak hunting */ - /* * We have many arrays of size 3. Whenever we have such an array, the * indices refer to one of the sides of the three-way merge. This is so @@ -305,8 +303,6 @@ struct merge_options_internal { * * these keys serve to intern all the path strings, which allows * us to do pointer comparison on directory names instead of * strcmp; we just have to be careful to use the interned strings. - * (Technically paths_to_free may track some strings that were - * removed from froms paths.) * * The values of paths: * * either a pointer to a merged_info, or a conflict_info struct @@ -349,18 +345,7 @@ struct merge_options_internal { * freed together too. Using a memory pool for these provides a * nice speedup. */ - struct mem_pool internal_pool; - struct mem_pool *pool; /* NULL, or pointer to internal_pool */ - - /* - * paths_to_free: additional list of strings to free - * - * If keys are removed from "paths", they are added to paths_to_free - * to ensure they are later freed. We avoid free'ing immediately since - * other places (e.g. conflict_info.pathnames[]) may still be - * referencing these paths. - */ - struct string_list paths_to_free; + struct mem_pool pool; /* * output: special messages and conflict notices for various paths @@ -539,19 +524,7 @@ static void clear_or_reinit_internal_opts(struct merge_options_internal *opti, void (*strset_clear_func)(struct strset *) = reinitialize ? strset_partial_clear : strset_clear; - if (opti->pool) - strmap_clear_func(&opti->paths, 0); - else { - /* - * We marked opti->paths with strdup_strings = 0, so that - * we wouldn't have to make another copy of the fullpath - * created by make_traverse_path from setup_path_info(). - * But, now that we've used it and have no other references - * to these strings, it is time to deallocate them. - */ - free_strmap_strings(&opti->paths); - strmap_clear_func(&opti->paths, 1); - } + strmap_clear_func(&opti->paths, 0); /* * All keys and values in opti->conflicted are a subset of those in @@ -560,20 +533,6 @@ static void clear_or_reinit_internal_opts(struct merge_options_internal *opti, */ strmap_clear_func(&opti->conflicted, 0); - if (!opti->pool) { - /* - * opti->paths_to_free is similar to opti->paths; we - * created it with strdup_strings = 0 to avoid making - * _another_ copy of the fullpath but now that we've used - * it and have no other references to these strings, it is - * time to deallocate them. We do so by temporarily - * setting strdup_strings to 1. - */ - opti->paths_to_free.strdup_strings = 1; - string_list_clear(&opti->paths_to_free, 0); - opti->paths_to_free.strdup_strings = 0; - } - if (opti->attr_index.cache_nr) /* true iff opt->renormalize */ discard_index(&opti->attr_index); @@ -623,11 +582,7 @@ static void clear_or_reinit_internal_opts(struct merge_options_internal *opti, strmap_clear(&opti->output, 0); } -#if USE_MEMORY_POOL - mem_pool_discard(&opti->internal_pool, 0); - if (!reinitialize) - opti->pool = NULL; -#endif + mem_pool_discard(&opti->pool, 0); /* Clean out callback_data as well. */ FREE_AND_NULL(renames->callback_data); @@ -693,12 +648,9 @@ static void path_msg(struct merge_options *opt, static struct diff_filespec *pool_alloc_filespec(struct mem_pool *pool, const char *path) { + /* Similar to alloc_filespec(), but allocate from pool and reuse path */ struct diff_filespec *spec; - if (!pool) - return alloc_filespec(path); - - /* Similar to alloc_filespec, but allocate from pool and reuse path */ spec = mem_pool_calloc(pool, 1, sizeof(*spec)); spec->path = (char*)path; /* spec won't modify it */ @@ -712,12 +664,9 @@ static struct diff_filepair *pool_diff_queue(struct mem_pool *pool, struct diff_filespec *one, struct diff_filespec *two) { + /* Same code as diff_queue(), except allocate from pool */ struct diff_filepair *dp; - if (!pool) - return diff_queue(queue, one, two); - - /* Same code as diff_queue, except allocate from pool */ dp = mem_pool_calloc(pool, 1, sizeof(*dp)); dp->one = one; dp->two = two; @@ -726,27 +675,6 @@ static struct diff_filepair *pool_diff_queue(struct mem_pool *pool, return dp; } -static void *pool_calloc(struct mem_pool *pool, size_t count, size_t size) -{ - if (!pool) - return xcalloc(count, size); - return mem_pool_calloc(pool, count, size); -} - -static void *pool_alloc(struct mem_pool *pool, size_t size) -{ - if (!pool) - return xmalloc(size); - return mem_pool_alloc(pool, size); -} - -static void *pool_strndup(struct mem_pool *pool, const char *str, size_t len) -{ - if (!pool) - return xstrndup(str, len); - return mem_pool_strndup(pool, str, len); -} - /* add a string to a strbuf, but converting "/" to "_" */ static void add_flattened_path(struct strbuf *out, const char *s) { @@ -875,9 +803,9 @@ static void setup_path_info(struct merge_options *opt, assert(!df_conflict || !resolved); /* df_conflict implies !resolved */ assert(resolved == (merged_version != NULL)); - mi = pool_calloc(opt->priv->pool, 1, - resolved ? sizeof(struct merged_info) : - sizeof(struct conflict_info)); + mi = mem_pool_calloc(&opt->priv->pool, 1, + resolved ? sizeof(struct merged_info) : + sizeof(struct conflict_info)); mi->directory_name = current_dir_name; mi->basename_offset = current_dir_name_len; mi->clean = !!resolved; @@ -924,7 +852,6 @@ static void add_pair(struct merge_options *opt, unsigned dir_rename_mask) { struct diff_filespec *one, *two; - struct mem_pool *pool = opt->priv->pool; struct rename_info *renames = &opt->priv->renames; int names_idx = is_add ? side : 0; @@ -975,11 +902,11 @@ static void add_pair(struct merge_options *opt, return; } - one = pool_alloc_filespec(pool, pathname); - two = pool_alloc_filespec(pool, pathname); + one = pool_alloc_filespec(&opt->priv->pool, pathname); + two = pool_alloc_filespec(&opt->priv->pool, pathname); fill_filespec(is_add ? two : one, &names[names_idx].oid, 1, names[names_idx].mode); - pool_diff_queue(pool, &renames->pairs[side], one, two); + pool_diff_queue(&opt->priv->pool, &renames->pairs[side], one, two); } static void collect_rename_info(struct merge_options *opt, @@ -1170,7 +1097,7 @@ static int collect_merge_info_callback(int n, len = traverse_path_len(info, p->pathlen); /* +1 in both of the following lines to include the NUL byte */ - fullpath = pool_alloc(opt->priv->pool, len + 1); + fullpath = mem_pool_alloc(&opt->priv->pool, len + 1); make_traverse_path(fullpath, len + 1, info, p->path, p->pathlen); /* @@ -1425,7 +1352,7 @@ static int handle_deferred_entries(struct merge_options *opt, copy = renames->deferred[side].possible_trivial_merges; strintmap_init_with_options(&renames->deferred[side].possible_trivial_merges, 0, - opt->priv->pool, + &opt->priv->pool, 0); strintmap_for_each_entry(©, &iter, entry) { const char *path = entry->key; @@ -2377,21 +2304,17 @@ static void apply_directory_rename_modifications(struct merge_options *opt, VERIFY_CI(ci); /* Find parent directories missing from opt->priv->paths */ - if (opt->priv->pool) { - cur_path = mem_pool_strdup(opt->priv->pool, new_path); - free((char*)new_path); - new_path = (char *)cur_path; - } else { - cur_path = new_path; - } + cur_path = mem_pool_strdup(&opt->priv->pool, new_path); + free((char*)new_path); + new_path = (char *)cur_path; while (1) { /* Find the parent directory of cur_path */ char *last_slash = strrchr(cur_path, '/'); if (last_slash) { - parent_name = pool_strndup(opt->priv->pool, - cur_path, - last_slash - cur_path); + parent_name = mem_pool_strndup(&opt->priv->pool, + cur_path, + last_slash - cur_path); } else { parent_name = opt->priv->toplevel_dir; break; @@ -2400,8 +2323,6 @@ static void apply_directory_rename_modifications(struct merge_options *opt, /* Look it up in opt->priv->paths */ entry = strmap_get_entry(&opt->priv->paths, parent_name); if (entry) { - if (!opt->priv->pool) - free((char*)parent_name); parent_name = entry->key; /* reuse known pointer */ break; } @@ -2428,16 +2349,6 @@ static void apply_directory_rename_modifications(struct merge_options *opt, parent_name = cur_dir; } - if (!opt->priv->pool) { - /* - * We are removing old_path from opt->priv->paths. - * old_path also will eventually need to be freed, but it - * may still be used by e.g. ci->pathnames. So, store it - * in another string-list for now. - */ - string_list_append(&opt->priv->paths_to_free, old_path); - } - assert(ci->filemask == 2 || ci->filemask == 4); assert(ci->dirmask == 0); strmap_remove(&opt->priv->paths, old_path, 0); @@ -2471,8 +2382,6 @@ static void apply_directory_rename_modifications(struct merge_options *opt, new_ci->stages[index].mode = ci->stages[index].mode; oidcpy(&new_ci->stages[index].oid, &ci->stages[index].oid); - if (!opt->priv->pool) - free(ci); ci = new_ci; } @@ -2888,7 +2797,6 @@ static void use_cached_pairs(struct merge_options *opt, { struct hashmap_iter iter; struct strmap_entry *entry; - struct mem_pool *pool = opt->priv->pool; /* * Add to side_pairs all entries from renames->cached_pairs[side_index]. @@ -2900,30 +2808,24 @@ static void use_cached_pairs(struct merge_options *opt, const char *new_name = entry->value; if (!new_name) new_name = old_name; - if (pool) { - /* - * cached_pairs has _copies* of old_name and new_name, - * because it has to persist across merges. When - * pool != NULL - * pool_alloc_filespec() will just re-use the existing - * filenames, which will also get re-used by - * opt->priv->paths if they become renames, and then - * get freed at the end of the merge, leaving the copy - * in cached_pairs dangling. Avoid this by making a - * copy here. - * - * When pool == NULL, pool_alloc_filespec() calls - * alloc_filespec(), which makes a copy; we don't want - * to add another. - */ - old_name = mem_pool_strdup(pool, old_name); - new_name = mem_pool_strdup(pool, new_name); - } + + /* + * cached_pairs has *copies* of old_name and new_name, + * because it has to persist across merges. Since + * pool_alloc_filespec() will just re-use the existing + * filenames, which will also get re-used by + * opt->priv->paths if they become renames, and then + * get freed at the end of the merge, that would leave + * the copy in cached_pairs dangling. Avoid this by + * making a copy here. + */ + old_name = mem_pool_strdup(&opt->priv->pool, old_name); + new_name = mem_pool_strdup(&opt->priv->pool, new_name); /* We don't care about oid/mode, only filenames and status */ - one = pool_alloc_filespec(pool, old_name); - two = pool_alloc_filespec(pool, new_name); - pool_diff_queue(pool, pairs, one, two); + one = pool_alloc_filespec(&opt->priv->pool, old_name); + two = pool_alloc_filespec(&opt->priv->pool, new_name); + pool_diff_queue(&opt->priv->pool, pairs, one, two); pairs->queue[pairs->nr-1]->status = entry->value ? 'R' : 'D'; } } @@ -3031,7 +2933,7 @@ static int detect_regular_renames(struct merge_options *opt, diff_queued_diff = renames->pairs[side_index]; trace2_region_enter("diff", "diffcore_rename", opt->repo); diffcore_rename_extended(&diff_opts, - opt->priv->pool, + &opt->priv->pool, &renames->relevant_sources[side_index], &renames->dirs_removed[side_index], &renames->dir_rename_count[side_index], @@ -3082,7 +2984,7 @@ static int collect_renames(struct merge_options *opt, if (p->status != 'A' && p->status != 'R') { possibly_cache_new_pair(renames, p, side_index, NULL); - pool_diff_free_filepair(opt->priv->pool, p); + pool_diff_free_filepair(&opt->priv->pool, p); continue; } @@ -3095,7 +2997,7 @@ static int collect_renames(struct merge_options *opt, possibly_cache_new_pair(renames, p, side_index, new_path); if (p->status != 'R' && !new_path) { - pool_diff_free_filepair(opt->priv->pool, p); + pool_diff_free_filepair(&opt->priv->pool, p); continue; } @@ -3213,7 +3115,7 @@ static int detect_and_process_renames(struct merge_options *opt, side_pairs = &renames->pairs[s]; for (i = 0; i < side_pairs->nr; ++i) { struct diff_filepair *p = side_pairs->queue[i]; - pool_diff_free_filepair(opt->priv->pool, p); + pool_diff_free_filepair(&opt->priv->pool, p); } } @@ -3226,7 +3128,7 @@ static int detect_and_process_renames(struct merge_options *opt, if (combined.nr) { int i; for (i = 0; i < combined.nr; i++) - pool_diff_free_filepair(opt->priv->pool, + pool_diff_free_filepair(&opt->priv->pool, combined.queue[i]); free(combined.queue); } @@ -3701,7 +3603,7 @@ static void process_entry(struct merge_options *opt, * the directory to remain here, so we need to move this * path to some new location. */ - new_ci = pool_calloc(opt->priv->pool, 1, sizeof(*new_ci)); + new_ci = mem_pool_calloc(&opt->priv->pool, 1, sizeof(*new_ci)); /* We don't really want new_ci->merged.result copied, but it'll * be overwritten below so it doesn't matter. We also don't @@ -3794,7 +3696,8 @@ static void process_entry(struct merge_options *opt, const char *a_path = NULL, *b_path = NULL; int rename_a = 0, rename_b = 0; - new_ci = pool_alloc(opt->priv->pool, sizeof(*new_ci)); + new_ci = mem_pool_alloc(&opt->priv->pool, + sizeof(*new_ci)); if (S_ISREG(a_mode)) rename_a = 1; @@ -3863,19 +3766,8 @@ static void process_entry(struct merge_options *opt, b_path = path; strmap_put(&opt->priv->paths, b_path, new_ci); - if (rename_a && rename_b) { + if (rename_a && rename_b) strmap_remove(&opt->priv->paths, path, 0); - /* - * We removed path from opt->priv->paths. path - * will also eventually need to be freed if not - * part of a memory pool...but it may still be - * used by e.g. ci->pathnames. So, store it in - * another string-list for now in that case. - */ - if (!opt->priv->pool) - string_list_append(&opt->priv->paths_to_free, - path); - } /* * Do special handling for b_path since process_entry() @@ -4482,13 +4374,8 @@ static void merge_start(struct merge_options *opt, struct merge_result *result) /* Initialization of various renames fields */ renames = &opt->priv->renames; -#if USE_MEMORY_POOL - mem_pool_init(&opt->priv->internal_pool, 0); - opt->priv->pool = &opt->priv->internal_pool; -#else - opt->priv->pool = NULL; -#endif - pool = opt->priv->pool; + mem_pool_init(&opt->priv->pool, 0); + pool = &opt->priv->pool; for (i = MERGE_SIDE1; i <= MERGE_SIDE2; i++) { strintmap_init_with_options(&renames->dirs_removed[i], NOT_RELEVANT, pool, 0); @@ -4525,15 +4412,13 @@ static void merge_start(struct merge_options *opt, struct merge_result *result) * Although we initialize opt->priv->paths with strdup_strings=0, * that's just to avoid making yet another copy of an allocated * string. Putting the entry into paths means we are taking - * ownership, so we will later free it. paths_to_free is similar. + * ownership, so we will later free it. * * In contrast, conflicted just has a subset of keys from paths, so * we don't want to free those (it'd be a duplicate free). */ strmap_init_with_options(&opt->priv->paths, pool, 0); strmap_init_with_options(&opt->priv->conflicted, pool, 0); - if (!opt->priv->pool) - string_list_init_nodup(&opt->priv->paths_to_free); /* * keys & strbufs in output will sometimes need to outlive "paths",