mirror of
https://github.com/git/git.git
synced 2024-05-12 21:06:08 +02:00
a56b9464cd
Revision walk will now use Bloom filters for commits to speed up revision walks for a particular path (for computing history for that path), if they are present in the commit-graph file. We load the Bloom filters during the prepare_revision_walk step, currently only when dealing with a single pathspec. Extending it to work with multiple pathspecs can be explored and built on top of this series in the future. While comparing trees in rev_compare_trees(), if the Bloom filter says that the file is not different between the two trees, we don't need to compute the expensive diff. This is where we get our performance gains. The other response of the Bloom filter is '`:maybe', in which case we fall back to the full diff calculation to determine if the path was changed in the commit. We do not try to use Bloom filters when the '--walk-reflogs' option is specified. The '--walk-reflogs' option does not walk the commit ancestry chain like the rest of the options. Incorporating the performance gains when walking reflog entries would add more complexity, and can be explored in a later series. Performance Gains: We tested the performance of `git log -- <path>` on the git repo, the linux and some internal large repos, with a variety of paths of varying depths. On the git and linux repos: - we observed a 2x to 5x speed up. On a large internal repo with files seated 6-10 levels deep in the tree: - we observed 10x to 20x speed ups, with some paths going up to 28 times faster. Helped-by: Derrick Stolee <dstolee@microsoft.com Helped-by: SZEDER Gábor <szeder.dev@gmail.com> Helped-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Garima Singh <garima.singh@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
90 lines
2.4 KiB
C
90 lines
2.4 KiB
C
#ifndef BLOOM_H
|
|
#define BLOOM_H
|
|
|
|
struct commit;
|
|
struct repository;
|
|
|
|
struct bloom_filter_settings {
|
|
/*
|
|
* The version of the hashing technique being used.
|
|
* We currently only support version = 1 which is
|
|
* the seeded murmur3 hashing technique implemented
|
|
* in bloom.c.
|
|
*/
|
|
uint32_t hash_version;
|
|
|
|
/*
|
|
* The number of times a path is hashed, i.e. the
|
|
* number of bit positions tht cumulatively
|
|
* determine whether a path is present in the
|
|
* Bloom filter.
|
|
*/
|
|
uint32_t num_hashes;
|
|
|
|
/*
|
|
* The minimum number of bits per entry in the Bloom
|
|
* filter. If the filter contains 'n' entries, then
|
|
* filter size is the minimum number of 8-bit words
|
|
* that contain n*b bits.
|
|
*/
|
|
uint32_t bits_per_entry;
|
|
};
|
|
|
|
#define DEFAULT_BLOOM_FILTER_SETTINGS { 1, 7, 10 }
|
|
#define BITS_PER_WORD 8
|
|
#define BLOOMDATA_CHUNK_HEADER_SIZE 3 * sizeof(uint32_t)
|
|
|
|
/*
|
|
* A bloom_filter struct represents a data segment to
|
|
* use when testing hash values. The 'len' member
|
|
* dictates how many entries are stored in
|
|
* 'data'.
|
|
*/
|
|
struct bloom_filter {
|
|
unsigned char *data;
|
|
size_t len;
|
|
};
|
|
|
|
/*
|
|
* A bloom_key represents the k hash values for a
|
|
* given string. These can be precomputed and
|
|
* stored in a bloom_key for re-use when testing
|
|
* against a bloom_filter. The number of hashes is
|
|
* given by the Bloom filter settings and is the same
|
|
* for all Bloom filters and keys interacting with
|
|
* the loaded version of the commit graph file and
|
|
* the Bloom data chunks.
|
|
*/
|
|
struct bloom_key {
|
|
uint32_t *hashes;
|
|
};
|
|
|
|
/*
|
|
* Calculate the murmur3 32-bit hash value for the given data
|
|
* using the given seed.
|
|
* Produces a uniformly distributed hash value.
|
|
* Not considered to be cryptographically secure.
|
|
* Implemented as described in https://en.wikipedia.org/wiki/MurmurHash#Algorithm
|
|
*/
|
|
uint32_t murmur3_seeded(uint32_t seed, const char *data, size_t len);
|
|
|
|
void fill_bloom_key(const char *data,
|
|
size_t len,
|
|
struct bloom_key *key,
|
|
const struct bloom_filter_settings *settings);
|
|
|
|
void add_key_to_filter(const struct bloom_key *key,
|
|
struct bloom_filter *filter,
|
|
const struct bloom_filter_settings *settings);
|
|
|
|
void init_bloom_filters(void);
|
|
|
|
struct bloom_filter *get_bloom_filter(struct repository *r,
|
|
struct commit *c,
|
|
int compute_if_not_present);
|
|
|
|
int bloom_filter_contains(const struct bloom_filter *filter,
|
|
const struct bloom_key *key,
|
|
const struct bloom_filter_settings *settings);
|
|
|
|
#endif |