From f4dbdfc4d514d4ea558703f5a234652dc68d83b2 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 3 Oct 2018 10:12:15 -0700 Subject: [PATCH 1/3] commit-graph: clean up leaked memory during write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The write_commit_graph() method in commit-graph.c leaks some lits and strings during execution. In addition, a list of strings is leaked in write_commit_graph_reachable(). Clean these up so our memory checking is cleaner. Further, if we use a list of pack-files to find the commits, we can leak the packed_git structs after scanning them for commits. Running the following commands demonstrates the leak before and the fix after: * valgrind --leak-check=full ./git commit-graph write --reachable * valgrind --leak-check=full ./git commit-graph write --stdin-packs Signed-off-by: Martin Ågren Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- commit-graph.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/commit-graph.c b/commit-graph.c index 2a24eb8b5a..ceca6026b0 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -693,11 +693,12 @@ static int add_ref_to_list(const char *refname, void write_commit_graph_reachable(const char *obj_dir, int append, int report_progress) { - struct string_list list; + struct string_list list = STRING_LIST_INIT_DUP; - string_list_init(&list, 1); for_each_ref(add_ref_to_list, &list); write_commit_graph(obj_dir, NULL, &list, append, report_progress); + + string_list_clear(&list, 0); } void write_commit_graph(const char *obj_dir, @@ -764,6 +765,7 @@ void write_commit_graph(const char *obj_dir, die(_("error opening index for %s"), packname.buf); for_each_object_in_pack(p, add_packed_commits, &oids, 0); close_pack(p); + free(p); } stop_progress(&oids.progress); strbuf_release(&packname); @@ -846,9 +848,11 @@ void write_commit_graph(const char *obj_dir, compute_generation_numbers(&commits, report_progress); graph_name = get_commit_graph_filename(obj_dir); - if (safe_create_leading_directories(graph_name)) + if (safe_create_leading_directories(graph_name)) { + UNLEAK(graph_name); die_errno(_("unable to create leading directories of %s"), graph_name); + } hold_lock_file_for_update(&lk, graph_name, LOCK_DIE_ON_ERROR); f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf); @@ -893,9 +897,9 @@ void write_commit_graph(const char *obj_dir, finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_FSYNC); commit_lock_file(&lk); + free(graph_name); + free(commits.list); free(oids.list); - oids.alloc = 0; - oids.nr = 0; } #define VERIFY_COMMIT_GRAPH_ERROR_HASH 2 From 0bfb48e67230393632b9b07d8ffceae94398e5b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20=C3=85gren?= Date: Wed, 3 Oct 2018 10:12:17 -0700 Subject: [PATCH 2/3] builtin/commit-graph.c: UNLEAK variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `graph_verify()`, `graph_read()` and `graph_write()` do the hard work of `cmd_commit_graph()`. As soon as these return, so does `cmd_commit_graph()`. `strbuf_getline()` may allocate memory in the strbuf, yet return EOF. We need to release the strbuf or UNLEAK it. Go for the latter since we are close to returning from `graph_write()`. `graph_write()` also fails to free the strings in the string list. They have been added to the list with `strdup_strings` set to 0. We could flip `strdup_strings` before clearing the list, which is our usual hack in situations like this. But since we are about to exit, let's just UNLEAK the whole string list instead. UNLEAK `graph` in `graph_verify`. While at it, and for consistency, UNLEAK in `graph_read()` as well, and remove an unnecessary UNLEAK just before dying. Signed-off-by: Martin Ågren Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- builtin/commit-graph.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c index bc0fa9ba52..66f12eb009 100644 --- a/builtin/commit-graph.c +++ b/builtin/commit-graph.c @@ -64,6 +64,7 @@ static int graph_verify(int argc, const char **argv) if (!graph) return 0; + UNLEAK(graph); return verify_commit_graph(the_repository, graph); } @@ -89,10 +90,8 @@ static int graph_read(int argc, const char **argv) graph_name = get_commit_graph_filename(opts.obj_dir); graph = load_commit_graph_one(graph_name); - if (!graph) { - UNLEAK(graph_name); + if (!graph) die("graph file %s does not exist", graph_name); - } FREE_AND_NULL(graph_name); @@ -115,7 +114,7 @@ static int graph_read(int argc, const char **argv) printf(" large_edges"); printf("\n"); - free_commit_graph(graph); + UNLEAK(graph); return 0; } @@ -166,6 +165,8 @@ static int graph_write(int argc, const char **argv) pack_indexes = &lines; if (opts.stdin_commits) commit_hex = &lines; + + UNLEAK(buf); } write_commit_graph(opts.obj_dir, @@ -174,7 +175,7 @@ static int graph_write(int argc, const char **argv) opts.append, 1); - string_list_clear(&lines, 0); + UNLEAK(lines); return 0; } From 53c36670e7f21342fb6017f39e0c91362e50dfe0 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 3 Oct 2018 10:12:19 -0700 Subject: [PATCH 3/3] commit-graph: reduce initial oid allocation While writing a commit-graph file, we store the full list of commits in a flat list. We use this list for sorting and ensuring we are closed under reachability. The initial allocation assumed that (at most) one in four objects is a commit. This is a dramatic over-count for many repos, especially large ones. Since we grow the repo dynamically, reduce this count by a factor of eight. We still set it to a minimum of 1024 before allocating. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- commit-graph.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commit-graph.c b/commit-graph.c index ceca6026b0..e773703e1d 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -720,7 +720,7 @@ void write_commit_graph(const char *obj_dir, struct progress *progress = NULL; oids.nr = 0; - oids.alloc = approximate_object_count() / 4; + oids.alloc = approximate_object_count() / 32; oids.progress = NULL; oids.progress_done = 0;