mirror/ git
1
0
Fork 0
mirror of https://github.com/git/git.git synced 2024-05-09 05:36:10 +02:00
Code

Merge branch 'ds/close-object-store' into maint 

The commit-graph file is now part of the "files that the runtime
may keep open file descriptors on, all of which would need to be
closed when done with the object store", and the file descriptor to
an existing commit-graph file now is closed before "gc" finalizes a
new instance to replace it.

* ds/close-object-store:
  packfile: rename close_all_packs to close_object_store
  packfile: close commit-graph in close_all_packs
  commit-graph: use raw_object_store when closing
  commit-graph: extract write_commit_graph_file()
  commit-graph: extract copy_oids_to_commits()
  commit-graph: extract count_distinct_commits()
  commit-graph: extract fill_oids_from_all_packs()
  commit-graph: extract fill_oids_from_commit_hex()
  commit-graph: extract fill_oids_from_packs()
  commit-graph: create write_commit_graph_context
  commit-graph: remove Future Work section
  commit-graph: collapse parameters into flags
  commit-graph: return with errors during write
  commit-graph: fix the_repository reference
This commit is contained in:
Junio C Hamano 2019-07-29 12:38:22 -07:00
parent 689204ca88 2d511cfc0b
commit dea6737bb7
19 changed files with 406 additions and 335 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
Documentation/technical/commit-graph.txt
View File

@ -127,23 +127,6 @@ Design Details
helpful for these clones, anyway. The commit-graph will not be read or
written when shallow commits are present.
Future Work
-----------
- After computing and storing generation numbers, we must make graph
walks aware of generation numbers to gain the performance benefits they
enable. This will mostly be accomplished by swapping a commit-date-ordered
priority queue with one ordered by generation number. The following
operations are important candidates:
- 'log --topo-order'
- 'tag --merged'
- A server could provide a commit-graph file as part of the network protocol
to avoid extra calculations by clients. This feature is only of benefit if
the user is willing to trust the file, because verifying the file is correct
is as hard as computing it from scratch.
Related Links
-------------
[0] https://bugs.chromium.org/p/git/issues/detail?id=8

2
builtin/am.c
View File

@ -1801,7 +1801,7 @@ static void am_run(struct am_state *state, int resume)
*/
if (!state->rebasing) {
am_destroy(state);
close_all_packs(the_repository->objects);
close_object_store(the_repository->objects);
run_command_v_opt(argv_gc_auto, RUN_GIT_CMD);
}
}

2
builtin/clone.c
View File

@ -1245,7 +1245,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
transport_disconnect(transport);
if (option_dissociate) {
close_all_packs(the_repository->objects);
close_object_store(the_repository->objects);
dissociate_from_references();
}

22
builtin/commit-graph.c
View File

@ -141,6 +141,8 @@ static int graph_write(int argc, const char **argv)
struct string_list *pack_indexes = NULL;
struct string_list *commit_hex = NULL;
struct string_list lines;
int result = 0;
unsigned int flags = COMMIT_GRAPH_PROGRESS;
static struct option builtin_commit_graph_write_options[] = {
OPT_STRING(0, "object-dir", &opts.obj_dir,
@ -165,13 +167,13 @@ static int graph_write(int argc, const char **argv)
die(_("use at most one of --reachable, --stdin-commits, or --stdin-packs"));
if (!opts.obj_dir)
opts.obj_dir = get_object_directory();
if (opts.append)
flags |= COMMIT_GRAPH_APPEND;
read_replace_refs = 0;
if (opts.reachable) {
write_commit_graph_reachable(opts.obj_dir, opts.append, 1);
return 0;
}
if (opts.reachable)
return write_commit_graph_reachable(opts.obj_dir, flags);
string_list_init(&lines, 0);
if (opts.stdin_packs || opts.stdin_commits) {
@ -188,14 +190,14 @@ static int graph_write(int argc, const char **argv)
UNLEAK(buf);
}
write_commit_graph(opts.obj_dir,
pack_indexes,
commit_hex,
opts.append,
1);
if (write_commit_graph(opts.obj_dir,
pack_indexes,
commit_hex,
flags))
result = 1;
UNLEAK(lines);
return 0;
return result;
}
int cmd_commit_graph(int argc, const char **argv, const char *prefix)

5
builtin/commit.c
View File

@ -1669,8 +1669,9 @@ int cmd_commit(int argc, const char **argv, const char *prefix)
"new_index file. Check that disk is not full and quota is\n"
"not exceeded, and then \"git reset HEAD\" to recover."));
if (git_env_bool(GIT_TEST_COMMIT_GRAPH, 0))
write_commit_graph_reachable(get_object_directory(), 0, 0);
if (git_env_bool(GIT_TEST_COMMIT_GRAPH, 0) &&
write_commit_graph_reachable(get_object_directory(), 0))
return 1;
repo_rerere(the_repository, 0);
run_command_v_opt(argv_gc_auto, RUN_GIT_CMD);

2
builtin/fetch.c
View File

@ -1672,7 +1672,7 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
string_list_clear(&list, 0);
close_all_packs(the_repository->objects);
close_object_store(the_repository->objects);
argv_array_pushl(&argv_gc_auto, "gc", "--auto", NULL);
if (verbosity < 0)

11
builtin/gc.c
View File

@ -653,7 +653,7 @@ int cmd_gc(int argc, const char **argv, const char *prefix)
gc_before_repack();
if (!repository_format_precious_objects) {
close_all_packs(the_repository->objects);
close_object_store(the_repository->objects);
if (run_command_v_opt(repack.argv, RUN_GIT_CMD))
die(FAILED_RUN, repack.argv[0]);
@ -681,13 +681,14 @@ int cmd_gc(int argc, const char **argv, const char *prefix)
report_garbage = report_pack_garbage;
reprepare_packed_git(the_repository);
if (pack_garbage.nr > 0) {
close_all_packs(the_repository->objects);
close_object_store(the_repository->objects);
clean_pack_garbage();
}
if (gc_write_commit_graph)
write_commit_graph_reachable(get_object_directory(), 0,
!quiet && !daemonized);
if (gc_write_commit_graph &&
write_commit_graph_reachable(get_object_directory(),
!quiet && !daemonized ? COMMIT_GRAPH_PROGRESS : 0))
return 1;
if (auto_gc && too_many_loose_objects())
warning(_("There are too many unreachable loose objects; "

2
builtin/merge.c
View File

@ -457,7 +457,7 @@ static void finish(struct commit *head_commit,
* We ignore errors in 'gc --auto', since the
* user should see them.
*/
close_all_packs(the_repository->objects);
close_object_store(the_repository->objects);
run_command_v_opt(argv_gc_auto, RUN_GIT_CMD);
}
}

2
builtin/rebase.c
View File

@ -742,7 +742,7 @@ static int finish_rebase(struct rebase_options *opts)
delete_ref(NULL, "REBASE_HEAD", NULL, REF_NO_DEREF);
apply_autostash(opts);
close_all_packs(the_repository->objects);
close_object_store(the_repository->objects);
/*
* We ignore errors in 'gc --auto', since the
* user should see them.

2
builtin/receive-pack.c
View File

@ -2043,7 +2043,7 @@ int cmd_receive_pack(int argc, const char **argv, const char *prefix)
proc.git_cmd = 1;
proc.argv = argv_gc_auto;
close_all_packs(the_repository->objects);
close_object_store(the_repository->objects);
if (!start_command(&proc)) {
if (use_sideband)
copy_to_sideband(proc.err, -1, NULL);

2
builtin/repack.c
View File

@ -422,7 +422,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
if (!names.nr && !po_args.quiet)
printf_ln(_("Nothing new to pack."));
close_all_packs(the_repository->objects);
close_object_store(the_repository->objects);
/*
* Ok we have prepared all new packfiles.

629
commit-graph.c
View File

@ -361,10 +361,10 @@ int generation_numbers_enabled(struct repository *r)
return !!first_generation;
}
void close_commit_graph(struct repository *r)
void close_commit_graph(struct raw_object_store *o)
{
free_commit_graph(r->objects->commit_graph);
r->objects->commit_graph = NULL;
free_commit_graph(o->commit_graph);
o->commit_graph = NULL;
}
static int bsearch_graph(struct commit_graph *g, struct object_id *oid, uint32_t *pos)
@ -525,14 +525,38 @@ struct tree *get_commit_tree_in_graph(struct repository *r, const struct commit
return get_commit_tree_in_graph_one(r, r->objects->commit_graph, c);
}
struct packed_commit_list {
struct commit **list;
int nr;
int alloc;
};
struct packed_oid_list {
struct object_id *list;
int nr;
int alloc;
};
struct write_commit_graph_context {
struct repository *r;
const char *obj_dir;
char *graph_name;
struct packed_oid_list oids;
struct packed_commit_list commits;
int num_extra_edges;
unsigned long approx_nr_objects;
struct progress *progress;
int progress_done;
uint64_t progress_cnt;
unsigned append:1,
report_progress:1;
};
static void write_graph_chunk_fanout(struct hashfile *f,
struct commit **commits,
int nr_commits,
struct progress *progress,
uint64_t *progress_cnt)
struct write_commit_graph_context *ctx)
{
int i, count = 0;
struct commit **list = commits;
struct commit **list = ctx->commits.list;
/*
* Write the first-level table (the list is sorted,
@ -540,10 +564,10 @@ static void write_graph_chunk_fanout(struct hashfile *f,
* having to do eight extra binary search iterations).
*/
for (i = 0; i < 256; i++) {
while (count < nr_commits) {
while (count < ctx->commits.nr) {
if ((*list)->object.oid.hash[0] != i)
break;
display_progress(progress, ++*progress_cnt);
display_progress(ctx->progress, ++ctx->progress_cnt);
count++;
list++;
}
@ -553,14 +577,12 @@ static void write_graph_chunk_fanout(struct hashfile *f,
}
static void write_graph_chunk_oids(struct hashfile *f, int hash_len,
struct commit **commits, int nr_commits,
struct progress *progress,
uint64_t *progress_cnt)
struct write_commit_graph_context *ctx)
{
struct commit **list = commits;
struct commit **list = ctx->commits.list;
int count;
for (count = 0; count < nr_commits; count++, list++) {
display_progress(progress, ++*progress_cnt);
for (count = 0; count < ctx->commits.nr; count++, list++) {
display_progress(ctx->progress, ++ctx->progress_cnt);
hashwrite(f, (*list)->object.oid.hash, (int)hash_len);
}
}
@ -572,19 +594,17 @@ static const unsigned char *commit_to_sha1(size_t index, void *table)
}
static void write_graph_chunk_data(struct hashfile *f, int hash_len,
struct commit **commits, int nr_commits,
struct progress *progress,
uint64_t *progress_cnt)
struct write_commit_graph_context *ctx)
{
struct commit **list = commits;
struct commit **last = commits + nr_commits;
struct commit **list = ctx->commits.list;
struct commit **last = ctx->commits.list + ctx->commits.nr;
uint32_t num_extra_edges = 0;
while (list < last) {
struct commit_list *parent;
int edge_value;
uint32_t packedDate[2];
display_progress(progress, ++*progress_cnt);
display_progress(ctx->progress, ++ctx->progress_cnt);
parse_commit_no_graph(*list);
hashwrite(f, get_commit_tree_oid(*list)->hash, hash_len);
@ -595,8 +615,8 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len,
edge_value = GRAPH_PARENT_NONE;
else {
edge_value = sha1_pos(parent->item->object.oid.hash,
commits,
nr_commits,
ctx->commits.list,
ctx->commits.nr,
commit_to_sha1);
if (edge_value < 0)
@ -616,8 +636,8 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len,
edge_value = GRAPH_EXTRA_EDGES_NEEDED | num_extra_edges;
else {
edge_value = sha1_pos(parent->item->object.oid.hash,
commits,
nr_commits,
ctx->commits.list,
ctx->commits.nr,
commit_to_sha1);
if (edge_value < 0)
BUG("missing parent %s for commit %s",
@ -649,19 +669,16 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len,
}
static void write_graph_chunk_extra_edges(struct hashfile *f,
struct commit **commits,
int nr_commits,
struct progress *progress,
uint64_t *progress_cnt)
struct write_commit_graph_context *ctx)
{
struct commit **list = commits;
struct commit **last = commits + nr_commits;
struct commit **list = ctx->commits.list;
struct commit **last = ctx->commits.list + ctx->commits.nr;
struct commit_list *parent;
while (list < last) {
int num_parents = 0;
display_progress(progress, ++*progress_cnt);
display_progress(ctx->progress, ++ctx->progress_cnt);
for (parent = (*list)->parents; num_parents < 3 && parent;
parent = parent->next)
@ -675,8 +692,8 @@ static void write_graph_chunk_extra_edges(struct hashfile *f,
/* Since num_parents > 2, this initializer is safe. */
for (parent = (*list)->parents->next; parent; parent = parent->next) {
int edge_value = sha1_pos(parent->item->object.oid.hash,
commits,
nr_commits,
ctx->commits.list,
ctx->commits.nr,
commit_to_sha1);
if (edge_value < 0)
@ -700,125 +717,111 @@ static int commit_compare(const void *_a, const void *_b)
return oidcmp(a, b);
}
struct packed_commit_list {
struct commit **list;
int nr;
int alloc;
};
struct packed_oid_list {
struct object_id *list;
int nr;
int alloc;
struct progress *progress;
int progress_done;
};
static int add_packed_commits(const struct object_id *oid,
struct packed_git *pack,
uint32_t pos,
void *data)
{
struct packed_oid_list *list = (struct packed_oid_list*)data;
struct write_commit_graph_context *ctx = (struct write_commit_graph_context*)data;
enum object_type type;
off_t offset = nth_packed_object_offset(pack, pos);
struct object_info oi = OBJECT_INFO_INIT;
if (list->progress)
display_progress(list->progress, ++list->progress_done);
if (ctx->progress)
display_progress(ctx->progress, ++ctx->progress_done);
oi.typep = &type;
if (packed_object_info(the_repository, pack, offset, &oi) < 0)
if (packed_object_info(ctx->r, pack, offset, &oi) < 0)
die(_("unable to get type of object %s"), oid_to_hex(oid));
if (type != OBJ_COMMIT)
return 0;
ALLOC_GROW(list->list, list->nr + 1, list->alloc);
oidcpy(&(list->list[list->nr]), oid);
list->nr++;
ALLOC_GROW(ctx->oids.list, ctx->oids.nr + 1, ctx->oids.alloc);
oidcpy(&(ctx->oids.list[ctx->oids.nr]), oid);
ctx->oids.nr++;
return 0;
}
static void add_missing_parents(struct packed_oid_list *oids, struct commit *commit)
static void add_missing_parents(struct write_commit_graph_context *ctx, struct commit *commit)
{
struct commit_list *parent;
for (parent = commit->parents; parent; parent = parent->next) {
if (!(parent->item->object.flags & UNINTERESTING)) {
ALLOC_GROW(oids->list, oids->nr + 1, oids->alloc);
oidcpy(&oids->list[oids->nr], &(parent->item->object.oid));
oids->nr++;
ALLOC_GROW(ctx->oids.list, ctx->oids.nr + 1, ctx->oids.alloc);
oidcpy(&ctx->oids.list[ctx->oids.nr], &(parent->item->object.oid));
ctx->oids.nr++;
parent->item->object.flags |= UNINTERESTING;
}
}
}
static void close_reachable(struct packed_oid_list *oids, int report_progress)
static void close_reachable(struct write_commit_graph_context *ctx)
{
int i;
struct commit *commit;
struct progress *progress = NULL;
if (report_progress)
progress = start_delayed_progress(
_("Loading known commits in commit graph"), oids->nr);
for (i = 0; i < oids->nr; i++) {
display_progress(progress, i + 1);
commit = lookup_commit(the_repository, &oids->list[i]);
if (ctx->report_progress)
ctx->progress = start_delayed_progress(
_("Loading known commits in commit graph"),
ctx->oids.nr);
for (i = 0; i < ctx->oids.nr; i++) {
display_progress(ctx->progress, i + 1);
commit = lookup_commit(ctx->r, &ctx->oids.list[i]);
if (commit)
commit->object.flags |= UNINTERESTING;
}
stop_progress(&progress);
stop_progress(&ctx->progress);
/*
* As this loop runs, oids->nr may grow, but not more
* As this loop runs, ctx->oids.nr may grow, but not more
* than the number of missing commits in the reachable
* closure.
*/
if (report_progress)
progress = start_delayed_progress(
_("Expanding reachable commits in commit graph"), oids->nr);
for (i = 0; i < oids->nr; i++) {
display_progress(progress, i + 1);
commit = lookup_commit(the_repository, &oids->list[i]);
if (ctx->report_progress)
ctx->progress = start_delayed_progress(
_("Expanding reachable commits in commit graph"),
ctx->oids.nr);
for (i = 0; i < ctx->oids.nr; i++) {
display_progress(ctx->progress, i + 1);
commit = lookup_commit(ctx->r, &ctx->oids.list[i]);
if (commit && !parse_commit_no_graph(commit))
add_missing_parents(oids, commit);
add_missing_parents(ctx, commit);
}
stop_progress(&progress);
stop_progress(&ctx->progress);
if (report_progress)
progress = start_delayed_progress(
_("Clearing commit marks in commit graph"), oids->nr);
for (i = 0; i < oids->nr; i++) {
display_progress(progress, i + 1);
commit = lookup_commit(the_repository, &oids->list[i]);
if (ctx->report_progress)
ctx->progress = start_delayed_progress(
_("Clearing commit marks in commit graph"),
ctx->oids.nr);
for (i = 0; i < ctx->oids.nr; i++) {
display_progress(ctx->progress, i + 1);
commit = lookup_commit(ctx->r, &ctx->oids.list[i]);
if (commit)
commit->object.flags &= ~UNINTERESTING;
}
stop_progress(&progress);
stop_progress(&ctx->progress);
}
static void compute_generation_numbers(struct packed_commit_list* commits,
int report_progress)
static void compute_generation_numbers(struct write_commit_graph_context *ctx)
{
int i;
struct commit_list *list = NULL;
struct progress *progress = NULL;
if (report_progress)
progress = start_progress(
_("Computing commit graph generation numbers"),
commits->nr);
for (i = 0; i < commits->nr; i++) {
display_progress(progress, i + 1);
if (commits->list[i]->generation != GENERATION_NUMBER_INFINITY &&
commits->list[i]->generation != GENERATION_NUMBER_ZERO)
if (ctx->report_progress)
ctx->progress = start_progress(
_("Computing commit graph generation numbers"),
ctx->commits.nr);
for (i = 0; i < ctx->commits.nr; i++) {
display_progress(ctx->progress, i + 1);
if (ctx->commits.list[i]->generation != GENERATION_NUMBER_INFINITY &&
ctx->commits.list[i]->generation != GENERATION_NUMBER_ZERO)
continue;
commit_list_insert(commits->list[i], &list);
commit_list_insert(ctx->commits.list[i], &list);
while (list) {
struct commit *current = list->item;
struct commit_list *parent;
@ -845,7 +848,7 @@ static void compute_generation_numbers(struct packed_commit_list* commits,
}
}
}
stop_progress(&progress);
stop_progress(&ctx->progress);
}
static int add_ref_to_list(const char *refname,
@ -858,207 +861,187 @@ static int add_ref_to_list(const char *refname,
return 0;
}
void write_commit_graph_reachable(const char *obj_dir, int append,
int report_progress)
int write_commit_graph_reachable(const char *obj_dir, unsigned int flags)
{
struct string_list list = STRING_LIST_INIT_DUP;
int result;
for_each_ref(add_ref_to_list, &list);
write_commit_graph(obj_dir, NULL, &list, append, report_progress);
result = write_commit_graph(obj_dir, NULL, &list,
flags);
string_list_clear(&list, 0);
return result;
}
void write_commit_graph(const char *obj_dir,
struct string_list *pack_indexes,
struct string_list *commit_hex,
int append, int report_progress)
static int fill_oids_from_packs(struct write_commit_graph_context *ctx,
struct string_list *pack_indexes)
{
struct packed_oid_list oids;
struct packed_commit_list commits;
struct hashfile *f;
uint32_t i, count_distinct = 0;
char *graph_name;
struct lock_file lk = LOCK_INIT;
uint32_t chunk_ids[5];
uint64_t chunk_offsets[5];
int num_chunks;
int num_extra_edges;
struct commit_list *parent;
struct progress *progress = NULL;
const unsigned hashsz = the_hash_algo->rawsz;
uint64_t progress_cnt = 0;
uint32_t i;
struct strbuf progress_title = STRBUF_INIT;
unsigned long approx_nr_objects;
struct strbuf packname = STRBUF_INIT;
int dirlen;
if (!commit_graph_compatible(the_repository))
return;
oids.nr = 0;
approx_nr_objects = approximate_object_count();
oids.alloc = approx_nr_objects / 32;
oids.progress = NULL;
oids.progress_done = 0;
if (append) {
prepare_commit_graph_one(the_repository, obj_dir);
if (the_repository->objects->commit_graph)
oids.alloc += the_repository->objects->commit_graph->num_commits;
strbuf_addf(&packname, "%s/pack/", ctx->obj_dir);
dirlen = packname.len;
if (ctx->report_progress) {
strbuf_addf(&progress_title,
Q_("Finding commits for commit graph in %d pack",
"Finding commits for commit graph in %d packs",
pack_indexes->nr),
pack_indexes->nr);
ctx->progress = start_delayed_progress(progress_title.buf, 0);
ctx->progress_done = 0;
}
if (oids.alloc < 1024)
oids.alloc = 1024;
ALLOC_ARRAY(oids.list, oids.alloc);
if (append && the_repository->objects->commit_graph) {
struct commit_graph *commit_graph =
the_repository->objects->commit_graph;
for (i = 0; i < commit_graph->num_commits; i++) {
const unsigned char *hash = commit_graph->chunk_oid_lookup +
commit_graph->hash_len * i;
hashcpy(oids.list[oids.nr++].hash, hash);
for (i = 0; i < pack_indexes->nr; i++) {
struct packed_git *p;
strbuf_setlen(&packname, dirlen);
strbuf_addstr(&packname, pack_indexes->items[i].string);
p = add_packed_git(packname.buf, packname.len, 1);
if (!p) {
error(_("error adding pack %s"), packname.buf);
return -1;
}
}
if (pack_indexes) {
struct strbuf packname = STRBUF_INIT;
int dirlen;
strbuf_addf(&packname, "%s/pack/", obj_dir);
dirlen = packname.len;
if (report_progress) {
strbuf_addf(&progress_title,
Q_("Finding commits for commit graph in %d pack",
"Finding commits for commit graph in %d packs",
pack_indexes->nr),
pack_indexes->nr);
oids.progress = start_delayed_progress(progress_title.buf, 0);
oids.progress_done = 0;
if (open_pack_index(p)) {
error(_("error opening index for %s"), packname.buf);
return -1;
}
for (i = 0; i < pack_indexes->nr; i++) {
struct packed_git *p;
strbuf_setlen(&packname, dirlen);
strbuf_addstr(&packname, pack_indexes->items[i].string);
p = add_packed_git(packname.buf, packname.len, 1);
if (!p)
die(_("error adding pack %s"), packname.buf);
if (open_pack_index(p))
die(_("error opening index for %s"), packname.buf);
for_each_object_in_pack(p, add_packed_commits, &oids,
FOR_EACH_OBJECT_PACK_ORDER);
close_pack(p);
free(p);
}
stop_progress(&oids.progress);
strbuf_reset(&progress_title);
strbuf_release(&packname);
for_each_object_in_pack(p, add_packed_commits, ctx,
FOR_EACH_OBJECT_PACK_ORDER);
close_pack(p);
free(p);
}
if (commit_hex) {
if (report_progress) {
strbuf_addf(&progress_title,
Q_("Finding commits for commit graph from %d ref",
"Finding commits for commit graph from %d refs",
commit_hex->nr),
commit_hex->nr);
progress = start_delayed_progress(progress_title.buf,
commit_hex->nr);
}
for (i = 0; i < commit_hex->nr; i++) {
const char *end;
struct object_id oid;
struct commit *result;
stop_progress(&ctx->progress);
strbuf_reset(&progress_title);
strbuf_release(&packname);
display_progress(progress, i + 1);
if (commit_hex->items[i].string &&
parse_oid_hex(commit_hex->items[i].string, &oid, &end))
continue;
return 0;
}
result = lookup_commit_reference_gently(the_repository, &oid, 1);
static void fill_oids_from_commit_hex(struct write_commit_graph_context *ctx,
struct string_list *commit_hex)
{
uint32_t i;
struct strbuf progress_title = STRBUF_INIT;
if (result) {
ALLOC_GROW(oids.list, oids.nr + 1, oids.alloc);
oidcpy(&oids.list[oids.nr], &(result->object.oid));
oids.nr++;
}
}
stop_progress(&progress);
strbuf_reset(&progress_title);
if (ctx->report_progress) {
strbuf_addf(&progress_title,
Q_("Finding commits for commit graph from %d ref",
"Finding commits for commit graph from %d refs",
commit_hex->nr),
commit_hex->nr);
ctx->progress = start_delayed_progress(
progress_title.buf,
commit_hex->nr);
}
for (i = 0; i < commit_hex->nr; i++) {
const char *end;
struct object_id oid;
struct commit *result;
if (!pack_indexes && !commit_hex) {
if (report_progress)
oids.progress = start_delayed_progress(
_("Finding commits for commit graph among packed objects"),
approx_nr_objects);
for_each_packed_object(add_packed_commits, &oids,
FOR_EACH_OBJECT_PACK_ORDER);
if (oids.progress_done < approx_nr_objects)
display_progress(oids.progress, approx_nr_objects);
stop_progress(&oids.progress);
}
close_reachable(&oids, report_progress);
if (report_progress)
progress = start_delayed_progress(
_("Counting distinct commits in commit graph"),
oids.nr);
display_progress(progress, 0); /* TODO: Measure QSORT() progress */
QSORT(oids.list, oids.nr, commit_compare);
count_distinct = 1;
for (i = 1; i < oids.nr; i++) {
display_progress(progress, i + 1);
if (!oideq(&oids.list[i - 1], &oids.list[i]))
count_distinct++;
}
stop_progress(&progress);
if (count_distinct >= GRAPH_EDGE_LAST_MASK)
die(_("the commit graph format cannot write %d commits"), count_distinct);
commits.nr = 0;
commits.alloc = count_distinct;
ALLOC_ARRAY(commits.list, commits.alloc);
num_extra_edges = 0;
if (report_progress)
progress = start_delayed_progress(
_("Finding extra edges in commit graph"),
oids.nr);
for (i = 0; i < oids.nr; i++) {
int num_parents = 0;
display_progress(progress, i + 1);
if (i > 0 && oideq(&oids.list[i - 1], &oids.list[i]))
display_progress(ctx->progress, i + 1);
if (commit_hex->items[i].string &&
parse_oid_hex(commit_hex->items[i].string, &oid, &end))
continue;
commits.list[commits.nr] = lookup_commit(the_repository, &oids.list[i]);
parse_commit_no_graph(commits.list[commits.nr]);
result = lookup_commit_reference_gently(ctx->r, &oid, 1);
for (parent = commits.list[commits.nr]->parents;
if (result) {
ALLOC_GROW(ctx->oids.list, ctx->oids.nr + 1, ctx->oids.alloc);
oidcpy(&ctx->oids.list[ctx->oids.nr], &(result->object.oid));
ctx->oids.nr++;
}
}
stop_progress(&ctx->progress);
strbuf_release(&progress_title);
}
static void fill_oids_from_all_packs(struct write_commit_graph_context *ctx)
{
if (ctx->report_progress)
ctx->progress = start_delayed_progress(
_("Finding commits for commit graph among packed objects"),
ctx->approx_nr_objects);
for_each_packed_object(add_packed_commits, ctx,
FOR_EACH_OBJECT_PACK_ORDER);
if (ctx->progress_done < ctx->approx_nr_objects)
display_progress(ctx->progress, ctx->approx_nr_objects);
stop_progress(&ctx->progress);
}
static uint32_t count_distinct_commits(struct write_commit_graph_context *ctx)
{
uint32_t i, count_distinct = 1;
if (ctx->report_progress)
ctx->progress = start_delayed_progress(
_("Counting distinct commits in commit graph"),
ctx->oids.nr);
display_progress(ctx->progress, 0); /* TODO: Measure QSORT() progress */
QSORT(ctx->oids.list, ctx->oids.nr, commit_compare);
for (i = 1; i < ctx->oids.nr; i++) {
display_progress(ctx->progress, i + 1);
if (!oideq(&ctx->oids.list[i - 1], &ctx->oids.list[i]))
count_distinct++;
}
stop_progress(&ctx->progress);
return count_distinct;
}
static void copy_oids_to_commits(struct write_commit_graph_context *ctx)
{
uint32_t i;
struct commit_list *parent;
ctx->num_extra_edges = 0;
if (ctx->report_progress)
ctx->progress = start_delayed_progress(
_("Finding extra edges in commit graph"),
ctx->oids.nr);
for (i = 0; i < ctx->oids.nr; i++) {
int num_parents = 0;
display_progress(ctx->progress, i + 1);
if (i > 0 && oideq(&ctx->oids.list[i - 1], &ctx->oids.list[i]))
continue;
ctx->commits.list[ctx->commits.nr] = lookup_commit(ctx->r, &ctx->oids.list[i]);
parse_commit_no_graph(ctx->commits.list[ctx->commits.nr]);
for (parent = ctx->commits.list[ctx->commits.nr]->parents;
parent; parent = parent->next)
num_parents++;
if (num_parents > 2)
num_extra_edges += num_parents - 1;
ctx->num_extra_edges += num_parents - 1;
commits.nr++;
ctx->commits.nr++;
}
num_chunks = num_extra_edges ? 4 : 3;
stop_progress(&progress);
stop_progress(&ctx->progress);
}
if (commits.nr >= GRAPH_EDGE_LAST_MASK)
die(_("too many commits to write graph"));
static int write_commit_graph_file(struct write_commit_graph_context *ctx)
{
uint32_t i;
struct hashfile *f;
struct lock_file lk = LOCK_INIT;
uint32_t chunk_ids[5];
uint64_t chunk_offsets[5];
const unsigned hashsz = the_hash_algo->rawsz;
struct strbuf progress_title = STRBUF_INIT;
int num_chunks = ctx->num_extra_edges ? 4 : 3;
compute_generation_numbers(&commits, report_progress);
graph_name = get_commit_graph_filename(obj_dir);
if (safe_create_leading_directories(graph_name)) {
UNLEAK(graph_name);
die_errno(_("unable to create leading directories of %s"),
graph_name);
ctx->graph_name = get_commit_graph_filename(ctx->obj_dir);
if (safe_create_leading_directories(ctx->graph_name)) {
UNLEAK(ctx->graph_name);
error(_("unable to create leading directories of %s"),
ctx->graph_name);
return -1;
}
hold_lock_file_for_update(&lk, graph_name, LOCK_DIE_ON_ERROR);
hold_lock_file_for_update(&lk, ctx->graph_name, LOCK_DIE_ON_ERROR);
f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf);
hashwrite_be32(f, GRAPH_SIGNATURE);
@ -1071,7 +1054,7 @@ void write_commit_graph(const char *obj_dir,
chunk_ids[0] = GRAPH_CHUNKID_OIDFANOUT;
chunk_ids[1] = GRAPH_CHUNKID_OIDLOOKUP;
chunk_ids[2] = GRAPH_CHUNKID_DATA;
if (num_extra_edges)
if (ctx->num_extra_edges)
chunk_ids[3] = GRAPH_CHUNKID_EXTRAEDGES;
else
chunk_ids[3] = 0;
@ -1079,9 +1062,9 @@ void write_commit_graph(const char *obj_dir,
chunk_offsets[0] = 8 + (num_chunks + 1) * GRAPH_CHUNKLOOKUP_WIDTH;
chunk_offsets[1] = chunk_offsets[0] + GRAPH_FANOUT_SIZE;
chunk_offsets[2] = chunk_offsets[1] + hashsz * commits.nr;
chunk_offsets[3] = chunk_offsets[2] + (hashsz + 16) * commits.nr;
chunk_offsets[4] = chunk_offsets[3] + 4 * num_extra_edges;
chunk_offsets[2] = chunk_offsets[1] + hashsz * ctx->commits.nr;
chunk_offsets[3] = chunk_offsets[2] + (hashsz + 16) * ctx->commits.nr;
chunk_offsets[4] = chunk_offsets[3] + 4 * ctx->num_extra_edges;
for (i = 0; i <= num_chunks; i++) {
uint32_t chunk_write[3];
@ -1092,31 +1075,113 @@ void write_commit_graph(const char *obj_dir,
hashwrite(f, chunk_write, 12);
}
if (report_progress) {
if (ctx->report_progress) {
strbuf_addf(&progress_title,
Q_("Writing out commit graph in %d pass",
"Writing out commit graph in %d passes",
num_chunks),
num_chunks);
progress = start_delayed_progress(
ctx->progress = start_delayed_progress(
progress_title.buf,
num_chunks * commits.nr);
num_chunks * ctx->commits.nr);
}
write_graph_chunk_fanout(f, commits.list, commits.nr, progress, &progress_cnt);
write_graph_chunk_oids(f, hashsz, commits.list, commits.nr, progress, &progress_cnt);
write_graph_chunk_data(f, hashsz, commits.list, commits.nr, progress, &progress_cnt);
if (num_extra_edges)
write_graph_chunk_extra_edges(f, commits.list, commits.nr, progress, &progress_cnt);
stop_progress(&progress);
write_graph_chunk_fanout(f, ctx);
write_graph_chunk_oids(f, hashsz, ctx);
write_graph_chunk_data(f, hashsz, ctx);
if (ctx->num_extra_edges)
write_graph_chunk_extra_edges(f, ctx);
stop_progress(&ctx->progress);
strbuf_release(&progress_title);
close_commit_graph(the_repository);
close_commit_graph(ctx->r->objects);
finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_FSYNC);
commit_lock_file(&lk);
free(graph_name);
free(commits.list);
free(oids.list);
return 0;
}
int write_commit_graph(const char *obj_dir,
struct string_list *pack_indexes,
struct string_list *commit_hex,
unsigned int flags)
{
struct write_commit_graph_context *ctx;
uint32_t i, count_distinct = 0;
int res = 0;
if (!commit_graph_compatible(the_repository))
return 0;
ctx = xcalloc(1, sizeof(struct write_commit_graph_context));
ctx->r = the_repository;
ctx->obj_dir = obj_dir;
ctx->append = flags & COMMIT_GRAPH_APPEND ? 1 : 0;
ctx->report_progress = flags & COMMIT_GRAPH_PROGRESS ? 1 : 0;
ctx->approx_nr_objects = approximate_object_count();
ctx->oids.alloc = ctx->approx_nr_objects / 32;
if (ctx->append) {
prepare_commit_graph_one(ctx->r, ctx->obj_dir);
if (ctx->r->objects->commit_graph)
ctx->oids.alloc += ctx->r->objects->commit_graph->num_commits;
}
if (ctx->oids.alloc < 1024)
ctx->oids.alloc = 1024;
ALLOC_ARRAY(ctx->oids.list, ctx->oids.alloc);
if (ctx->append && ctx->r->objects->commit_graph) {
struct commit_graph *g = ctx->r->objects->commit_graph;
for (i = 0; i < g->num_commits; i++) {
const unsigned char *hash = g->chunk_oid_lookup + g->hash_len * i;
hashcpy(ctx->oids.list[ctx->oids.nr++].hash, hash);
}
}
if (pack_indexes) {
if ((res = fill_oids_from_packs(ctx, pack_indexes)))
goto cleanup;
}
if (commit_hex)
fill_oids_from_commit_hex(ctx, commit_hex);
if (!pack_indexes && !commit_hex)
fill_oids_from_all_packs(ctx);
close_reachable(ctx);
count_distinct = count_distinct_commits(ctx);
if (count_distinct >= GRAPH_EDGE_LAST_MASK) {
error(_("the commit graph format cannot write %d commits"), count_distinct);
res = -1;
goto cleanup;
}
ctx->commits.alloc = count_distinct;
ALLOC_ARRAY(ctx->commits.list, ctx->commits.alloc);
copy_oids_to_commits(ctx);
if (ctx->commits.nr >= GRAPH_EDGE_LAST_MASK) {
error(_("too many commits to write graph"));
res = -1;
goto cleanup;
}
compute_generation_numbers(ctx);
res = write_commit_graph_file(ctx);
cleanup:
free(ctx->graph_name);
free(ctx->commits.list);
free(ctx->oids.list);
free(ctx);
return res;
}
#define VERIFY_COMMIT_GRAPH_ERROR_HASH 2

22
commit-graph.h
View File

@ -65,16 +65,24 @@ struct commit_graph *parse_commit_graph(void *graph_map, int fd,
*/
int generation_numbers_enabled(struct repository *r);
void write_commit_graph_reachable(const char *obj_dir, int append,
int report_progress);
void write_commit_graph(const char *obj_dir,
struct string_list *pack_indexes,
struct string_list *commit_hex,
int append, int report_progress);
#define COMMIT_GRAPH_APPEND (1 << 0)
#define COMMIT_GRAPH_PROGRESS (1 << 1)
/*
* The write_commit_graph* methods return zero on success
* and a negative value on failure. Note that if the repository
* is not compatible with the commit-graph feature, then the
* methods will return 0 without writing a commit-graph.
*/
int write_commit_graph_reachable(const char *obj_dir, unsigned int flags);
int write_commit_graph(const char *obj_dir,
struct string_list *pack_indexes,
struct string_list *commit_hex,
unsigned int flags);
int verify_commit_graph(struct repository *r, struct commit_graph *g);
void close_commit_graph(struct repository *);
void close_commit_graph(struct raw_object_store *);
void free_commit_graph(struct commit_graph *);
#endif

2
commit.c
View File

@ -449,7 +449,7 @@ int parse_commit_buffer(struct repository *r, struct commit *item, const void *b
item->date = parse_commit_date(bufptr, tail);
if (check_graph)
load_commit_graph_info(the_repository, item);
load_commit_graph_info(r, item);
return 0;
}

2
object.c
View File

@ -517,7 +517,7 @@ void raw_object_store_clear(struct raw_object_store *o)
o->loaded_alternates = 0;
INIT_LIST_HEAD(&o->packed_git_mru);
close_all_packs(o);
close_object_store(o);
o->packed_git = NULL;
}

5
packfile.c
View File

@ -16,6 +16,7 @@
#include "tree.h"
#include "object-store.h"
#include "midx.h"
#include "commit-graph.h"
char *odb_pack_name(struct strbuf *buf,
const unsigned char *sha1,
@ -336,7 +337,7 @@ void close_pack(struct packed_git *p)
close_pack_index(p);
}
void close_all_packs(struct raw_object_store *o)
void close_object_store(struct raw_object_store *o)
{
struct packed_git *p;
@ -350,6 +351,8 @@ void close_all_packs(struct raw_object_store *o)
close_midx(o->multi_pack_index);
o->multi_pack_index = NULL;
}
close_commit_graph(o);
}
/*

2
packfile.h
View File

@ -90,7 +90,7 @@ uint32_t get_pack_fanout(struct packed_git *p, uint32_t value);
unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *);
void close_pack_windows(struct packed_git *);
void close_pack(struct packed_git *);
void close_all_packs(struct raw_object_store *o);
void close_object_store(struct raw_object_store *o);
void unuse_pack(struct pack_window **);
void clear_delta_base_cache(void);
struct packed_git *add_packed_git(const char *path, size_t path_len, int local);

8
t/t5318-commit-graph.sh
View File

@ -23,6 +23,14 @@ test_expect_success 'write graph with no packs' '
test_path_is_file info/commit-graph
'
test_expect_success 'close with correct error on bad input' '
cd "$TRASH_DIRECTORY/full" &&
echo doesnotexist >in &&
{ git commit-graph write --stdin-packs <in 2>stderr; ret=$?; } &&
test "$ret" = 1 &&
test_i18ngrep "error adding pack" stderr
'
test_expect_success 'create commits and repack' '
cd "$TRASH_DIRECTORY/full" &&
for i in $(test_seq 3)

2
upload-pack.c
View File

@ -722,7 +722,7 @@ static void deepen_by_rev_list(struct packet_writer *writer, int ac,
{
struct commit_list *result;
close_commit_graph(the_repository);
close_commit_graph(the_repository->objects);
result = get_shallow_commits_by_rev_list(ac, av, SHALLOW, NOT_SHALLOW);
send_shallow(writer, result);
free_commit_list(result);