mirror of
https://github.com/git/git.git
synced 2024-05-12 16:26:08 +02:00
Merge branch 'ab/pcre-jit-fixes'
A few simplification and bugfixes to PCRE interface. * ab/pcre-jit-fixes: grep: under --debug, show whether PCRE JIT is enabled grep: do not enter PCRE2_UTF mode on fixed matching grep: stess test PCRE v2 on invalid UTF-8 data grep: create a "is_fixed" member in "grep_pat" grep: consistently use "p->fixed" in compile_regexp() grep: stop using a custom JIT stack with PCRE v1 grep: stop "using" a custom JIT stack with PCRE v2 grep: remove overly paranoid BUG(...) code grep: use PCRE v2 for optimized fixed-string search grep: remove the kwset optimization grep: drop support for \0 in --fixed-strings <pattern> grep: make the behavior for NUL-byte in patterns sane grep tests: move binary pattern tests into their own file grep tests: move "grep binary" alongside the rest grep: inline the return value of a function call used only once t4210: skip more command-line encoding tests on MinGW grep: don't use PCRE2?_UTF8 with "log --encoding=<non-utf8>" log tests: test regex backends in "--encode=<enc>" tests
This commit is contained in:
Junio C Hamano
commit
a73f91774c
|
|
@ -271,6 +271,23 @@ providing this option will cause it to die.
|
|||
|
||||
-f <file>::
|
||||
Read patterns from <file>, one per line.
|
||||
+
|
||||
Passing the pattern via <file> allows for providing a search pattern
|
||||
containing a \0.
|
||||
+
|
||||
Not all pattern types support patterns containing \0. Git will error
|
||||
out if a given pattern type can't support such a pattern. The
|
||||
`--perl-regexp` pattern type when compiled against the PCRE v2 backend
|
||||
has the widest support for these types of patterns.
|
||||
+
|
||||
In versions of Git before 2.23.0 patterns containing \0 would be
|
||||
silently considered fixed. This was never documented, there were also
|
||||
odd and undocumented interactions between e.g. non-ASCII patterns
|
||||
containing \0 and `--ignore-case`.
|
||||
+
|
||||
In future versions we may learn to support patterns containing \0 for
|
||||
more search backends, until then we'll die when the pattern type in
|
||||
question doesn't support them.
|
||||
|
||||
-e::
|
||||
The next parameter is the pattern. This option has to be
|
||||
|
|
|
|||
176
grep.c
176
grep.c
|
|
@ -368,18 +368,6 @@ static int is_fixed(const char *s, size_t len)
|
|||
return 1;
|
||||
}
|
||||
|
||||
static int has_null(const char *s, size_t len)
|
||||
{
|
||||
/*
|
||||
* regcomp cannot accept patterns with NULs so when using it
|
||||
* we consider any pattern containing a NUL fixed.
|
||||
*/
|
||||
if (memchr(s, 0, len))
|
||||
return 1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
#ifdef USE_LIBPCRE1
|
||||
static void compile_pcre1_regexp(struct grep_pat *p, const struct grep_opt *opt)
|
||||
{
|
||||
|
|
@ -388,11 +376,11 @@ static void compile_pcre1_regexp(struct grep_pat *p, const struct grep_opt *opt)
|
|||
int options = PCRE_MULTILINE;
|
||||
|
||||
if (opt->ignore_case) {
|
||||
if (has_non_ascii(p->pattern))
|
||||
if (!opt->ignore_locale && has_non_ascii(p->pattern))
|
||||
p->pcre1_tables = pcre_maketables();
|
||||
options |= PCRE_CASELESS;
|
||||
}
|
||||
if (is_utf8_locale() && has_non_ascii(p->pattern))
|
||||
if (!opt->ignore_locale && is_utf8_locale() && has_non_ascii(p->pattern))
|
||||
options |= PCRE_UTF8;
|
||||
|
||||
p->pcre1_regexp = pcre_compile(p->pattern, options, &error, &erroffset,
|
||||
|
|
@ -406,15 +394,8 @@ static void compile_pcre1_regexp(struct grep_pat *p, const struct grep_opt *opt)
|
|||
|
||||
#ifdef GIT_PCRE1_USE_JIT
|
||||
pcre_config(PCRE_CONFIG_JIT, &p->pcre1_jit_on);
|
||||
if (p->pcre1_jit_on == 1) {
|
||||
p->pcre1_jit_stack = pcre_jit_stack_alloc(1, 1024 * 1024);
|
||||
if (!p->pcre1_jit_stack)
|
||||
die("Couldn't allocate PCRE JIT stack");
|
||||
pcre_assign_jit_stack(p->pcre1_extra_info, NULL, p->pcre1_jit_stack);
|
||||
} else if (p->pcre1_jit_on != 0) {
|
||||
BUG("The pcre1_jit_on variable should be 0 or 1, not %d",
|
||||
p->pcre1_jit_on);
|
||||
}
|
||||
if (opt->debug)
|
||||
fprintf(stderr, "pcre1_jit_on=%d\n", p->pcre1_jit_on);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
|
@ -426,18 +407,9 @@ static int pcre1match(struct grep_pat *p, const char *line, const char *eol,
|
|||
if (eflags & REG_NOTBOL)
|
||||
flags |= PCRE_NOTBOL;
|
||||
|
||||
#ifdef GIT_PCRE1_USE_JIT
|
||||
if (p->pcre1_jit_on) {
|
||||
ret = pcre_jit_exec(p->pcre1_regexp, p->pcre1_extra_info, line,
|
||||
eol - line, 0, flags, ovector,
|
||||
ARRAY_SIZE(ovector), p->pcre1_jit_stack);
|
||||
} else
|
||||
#endif
|
||||
{
|
||||
ret = pcre_exec(p->pcre1_regexp, p->pcre1_extra_info, line,
|
||||
eol - line, 0, flags, ovector,
|
||||
ARRAY_SIZE(ovector));
|
||||
}
|
||||
ret = pcre_exec(p->pcre1_regexp, p->pcre1_extra_info, line,
|
||||
eol - line, 0, flags, ovector,
|
||||
ARRAY_SIZE(ovector));
|
||||
|
||||
if (ret < 0 && ret != PCRE_ERROR_NOMATCH)
|
||||
die("pcre_exec failed with error code %d", ret);
|
||||
|
|
@ -454,14 +426,11 @@ static void free_pcre1_regexp(struct grep_pat *p)
|
|||
{
|
||||
pcre_free(p->pcre1_regexp);
|
||||
#ifdef GIT_PCRE1_USE_JIT
|
||||
if (p->pcre1_jit_on) {
|
||||
if (p->pcre1_jit_on)
|
||||
pcre_free_study(p->pcre1_extra_info);
|
||||
pcre_jit_stack_free(p->pcre1_jit_stack);
|
||||
} else
|
||||
else
|
||||
#endif
|
||||
{
|
||||
pcre_free(p->pcre1_extra_info);
|
||||
}
|
||||
pcre_free((void *)p->pcre1_tables);
|
||||
}
|
||||
#else /* !USE_LIBPCRE1 */
|
||||
|
|
@ -498,14 +467,15 @@ static void compile_pcre2_pattern(struct grep_pat *p, const struct grep_opt *opt
|
|||
p->pcre2_compile_context = NULL;
|
||||
|
||||
if (opt->ignore_case) {
|
||||
if (has_non_ascii(p->pattern)) {
|
||||
if (!opt->ignore_locale && has_non_ascii(p->pattern)) {
|
||||
character_tables = pcre2_maketables(NULL);
|
||||
p->pcre2_compile_context = pcre2_compile_context_create(NULL);
|
||||
pcre2_set_character_tables(p->pcre2_compile_context, character_tables);
|
||||
}
|
||||
options |= PCRE2_CASELESS;
|
||||
}
|
||||
if (is_utf8_locale() && has_non_ascii(p->pattern))
|
||||
if (!opt->ignore_locale && is_utf8_locale() && has_non_ascii(p->pattern) &&
|
||||
!(!opt->ignore_case && (p->fixed || p->is_fixed)))
|
||||
options |= PCRE2_UTF;
|
||||
|
||||
p->pcre2_pattern = pcre2_compile((PCRE2_SPTR)p->pattern,
|
||||
|
|
@ -522,7 +492,9 @@ static void compile_pcre2_pattern(struct grep_pat *p, const struct grep_opt *opt
|
|||
}
|
||||
|
||||
pcre2_config(PCRE2_CONFIG_JIT, &p->pcre2_jit_on);
|
||||
if (p->pcre2_jit_on == 1) {
|
||||
if (opt->debug)
|
||||
fprintf(stderr, "pcre2_jit_on=%d\n", p->pcre2_jit_on);
|
||||
if (p->pcre2_jit_on) {
|
||||
jitret = pcre2_jit_compile(p->pcre2_pattern, PCRE2_JIT_COMPLETE);
|
||||
if (jitret)
|
||||
die("Couldn't JIT the PCRE2 pattern '%s', got '%d'\n", p->pattern, jitret);
|
||||
|
|
@ -547,19 +519,11 @@ static void compile_pcre2_pattern(struct grep_pat *p, const struct grep_opt *opt
|
|||
BUG("pcre2_pattern_info() failed: %d", patinforet);
|
||||
if (jitsizearg == 0) {
|
||||
p->pcre2_jit_on = 0;
|
||||
if (opt->debug)
|
||||
fprintf(stderr, "pcre2_jit_on=%d: (*NO_JIT) in regex\n",
|
||||
p->pcre2_jit_on);
|
||||
return;
|
||||
}
|
||||
|
||||
p->pcre2_jit_stack = pcre2_jit_stack_create(1, 1024 * 1024, NULL);
|
||||
if (!p->pcre2_jit_stack)
|
||||
die("Couldn't allocate PCRE2 JIT stack");
|
||||
p->pcre2_match_context = pcre2_match_context_create(NULL);
|
||||
if (!p->pcre2_match_context)
|
||||
die("Couldn't allocate PCRE2 match context");
|
||||
pcre2_jit_stack_assign(p->pcre2_match_context, NULL, p->pcre2_jit_stack);
|
||||
} else if (p->pcre2_jit_on != 0) {
|
||||
BUG("The pcre2_jit_on variable should be 0 or 1, not %d",
|
||||
p->pcre2_jit_on);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -603,8 +567,6 @@ static void free_pcre2_pattern(struct grep_pat *p)
|
|||
pcre2_compile_context_free(p->pcre2_compile_context);
|
||||
pcre2_code_free(p->pcre2_pattern);
|
||||
pcre2_match_data_free(p->pcre2_match_data);
|
||||
pcre2_jit_stack_free(p->pcre2_jit_stack);
|
||||
pcre2_match_context_free(p->pcre2_match_context);
|
||||
}
|
||||
#else /* !USE_LIBPCRE2 */
|
||||
static void compile_pcre2_pattern(struct grep_pat *p, const struct grep_opt *opt)
|
||||
|
|
@ -626,7 +588,6 @@ static int pcre2match(struct grep_pat *p, const char *line, const char *eol,
|
|||
static void free_pcre2_pattern(struct grep_pat *p)
|
||||
{
|
||||
}
|
||||
#endif /* !USE_LIBPCRE2 */
|
||||
|
||||
static void compile_fixed_regexp(struct grep_pat *p, struct grep_opt *opt)
|
||||
{
|
||||
|
|
@ -647,46 +608,66 @@ static void compile_fixed_regexp(struct grep_pat *p, struct grep_opt *opt)
|
|||
compile_regexp_failed(p, errbuf);
|
||||
}
|
||||
}
|
||||
#endif /* !USE_LIBPCRE2 */
|
||||
|
||||
static void compile_regexp(struct grep_pat *p, struct grep_opt *opt)
|
||||
{
|
||||
int ascii_only;
|
||||
int err;
|
||||
int regflags = REG_NEWLINE;
|
||||
|
||||
p->word_regexp = opt->word_regexp;
|
||||
p->ignore_case = opt->ignore_case;
|
||||
ascii_only = !has_non_ascii(p->pattern);
|
||||
p->fixed = opt->fixed;
|
||||
|
||||
/*
|
||||
* Even when -F (fixed) asks us to do a non-regexp search, we
|
||||
* may not be able to correctly case-fold when -i
|
||||
* (ignore-case) is asked (in which case, we'll synthesize a
|
||||
* regexp to match the pattern that matches regexp special
|
||||
* characters literally, while ignoring case differences). On
|
||||
* the other hand, even without -F, if the pattern does not
|
||||
* have any regexp special characters and there is no need for
|
||||
* case-folding search, we can internally turn it into a
|
||||
* simple string match using kws. p->fixed tells us if we
|
||||
* want to use kws.
|
||||
*/
|
||||
if (opt->fixed ||
|
||||
has_null(p->pattern, p->patternlen) ||
|
||||
is_fixed(p->pattern, p->patternlen))
|
||||
p->fixed = !p->ignore_case || ascii_only;
|
||||
if (memchr(p->pattern, 0, p->patternlen) && !opt->pcre2)
|
||||
die(_("given pattern contains NULL byte (via -f <file>). This is only supported with -P under PCRE v2"));
|
||||
|
||||
if (p->fixed) {
|
||||
p->kws = kwsalloc(p->ignore_case ? tolower_trans_tbl : NULL);
|
||||
kwsincr(p->kws, p->pattern, p->patternlen);
|
||||
kwsprep(p->kws);
|
||||
return;
|
||||
} else if (opt->fixed) {
|
||||
/*
|
||||
* We come here when the pattern has the non-ascii
|
||||
* characters we cannot case-fold, and asked to
|
||||
* ignore-case.
|
||||
*/
|
||||
p->is_fixed = is_fixed(p->pattern, p->patternlen);
|
||||
#ifdef USE_LIBPCRE2
|
||||
if (!p->fixed && !p->is_fixed) {
|
||||
const char *no_jit = "(*NO_JIT)";
|
||||
const int no_jit_len = strlen(no_jit);
|
||||
if (starts_with(p->pattern, no_jit) &&
|
||||
is_fixed(p->pattern + no_jit_len,
|
||||
p->patternlen - no_jit_len))
|
||||
p->is_fixed = 1;
|
||||
}
|
||||
#endif
|
||||
if (p->fixed || p->is_fixed) {
|
||||
#ifdef USE_LIBPCRE2
|
||||
opt->pcre2 = 1;
|
||||
if (p->is_fixed) {
|
||||
compile_pcre2_pattern(p, opt);
|
||||
} else {
|
||||
/*
|
||||
* E.g. t7811-grep-open.sh relies on the
|
||||
* pattern being restored.
|
||||
*/
|
||||
char *old_pattern = p->pattern;
|
||||
size_t old_patternlen = p->patternlen;
|
||||
struct strbuf sb = STRBUF_INIT;
|
||||
|
||||
/*
|
||||
* There is the PCRE2_LITERAL flag, but it's
|
||||
* only in PCRE v2 10.30 and later. Needing to
|
||||
* ifdef our way around that and dealing with
|
||||
* it + PCRE2_MULTILINE being an error is more
|
||||
* complex than just quoting this ourselves.
|
||||
*/
|
||||
strbuf_add(&sb, "\\Q", 2);
|
||||
strbuf_add(&sb, p->pattern, p->patternlen);
|
||||
strbuf_add(&sb, "\\E", 2);
|
||||
|
||||
p->pattern = sb.buf;
|
||||
p->patternlen = sb.len;
|
||||
compile_pcre2_pattern(p, opt);
|
||||
p->pattern = old_pattern;
|
||||
p->patternlen = old_patternlen;
|
||||
strbuf_release(&sb);
|
||||
}
|
||||
#else /* !USE_LIBPCRE2 */
|
||||
compile_fixed_regexp(p, opt);
|
||||
#endif /* !USE_LIBPCRE2 */
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -1053,9 +1034,7 @@ void free_grep_patterns(struct grep_opt *opt)
|
|||
case GREP_PATTERN: /* atom */
|
||||
case GREP_PATTERN_HEAD:
|
||||
case GREP_PATTERN_BODY:
|
||||
if (p->kws)
|
||||
kwsfree(p->kws);
|
||||
else if (p->pcre1_regexp)
|
||||
if (p->pcre1_regexp)
|
||||
free_pcre1_regexp(p);
|
||||
else if (p->pcre2_pattern)
|
||||
free_pcre2_pattern(p);
|
||||
|
|
@ -1115,29 +1094,12 @@ static void show_name(struct grep_opt *opt, const char *name)
|
|||
opt->output(opt, opt->null_following_name ? "\0" : "\n", 1);
|
||||
}
|
||||
|
||||
static int fixmatch(struct grep_pat *p, char *line, char *eol,
|
||||
regmatch_t *match)
|
||||
{
|
||||
struct kwsmatch kwsm;
|
||||
size_t offset = kwsexec(p->kws, line, eol - line, &kwsm);
|
||||
if (offset == -1) {
|
||||
match->rm_so = match->rm_eo = -1;
|
||||
return REG_NOMATCH;
|
||||
} else {
|
||||
match->rm_so = offset;
|
||||
match->rm_eo = match->rm_so + kwsm.size[0];
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
static int patmatch(struct grep_pat *p, char *line, char *eol,
|
||||
regmatch_t *match, int eflags)
|
||||
{
|
||||
int hit;
|
||||
|
||||
if (p->fixed)
|
||||
hit = !fixmatch(p, line, eol, match);
|
||||
else if (p->pcre1_regexp)
|
||||
if (p->pcre1_regexp)
|
||||
hit = !pcre1match(p, line, eol, match, eflags);
|
||||
else if (p->pcre2_pattern)
|
||||
hit = !pcre2match(p, line, eol, match, eflags);
|
||||
|
|
|
|||
13
grep.h
13
grep.h
|
|
@ -17,13 +17,9 @@
|
|||
#ifndef GIT_PCRE_STUDY_JIT_COMPILE
|
||||
#define GIT_PCRE_STUDY_JIT_COMPILE 0
|
||||
#endif
|
||||
#if PCRE_MAJOR <= 8 && PCRE_MINOR < 20
|
||||
typedef int pcre_jit_stack;
|
||||
#endif
|
||||
#else
|
||||
typedef int pcre;
|
||||
typedef int pcre_extra;
|
||||
typedef int pcre_jit_stack;
|
||||
#endif
|
||||
#ifdef USE_LIBPCRE2
|
||||
#define PCRE2_CODE_UNIT_WIDTH 8
|
||||
|
|
@ -32,10 +28,7 @@ typedef int pcre_jit_stack;
|
|||
typedef int pcre2_code;
|
||||
typedef int pcre2_match_data;
|
||||
typedef int pcre2_compile_context;
|
||||
typedef int pcre2_match_context;
|
||||
typedef int pcre2_jit_stack;
|
||||
#endif
|
||||
#include "kwset.h"
|
||||
#include "thread-utils.h"
|
||||
#include "userdiff.h"
|
||||
|
||||
|
|
@ -91,17 +84,14 @@ struct grep_pat {
|
|||
regex_t regexp;
|
||||
pcre *pcre1_regexp;
|
||||
pcre_extra *pcre1_extra_info;
|
||||
pcre_jit_stack *pcre1_jit_stack;
|
||||
const unsigned char *pcre1_tables;
|
||||
int pcre1_jit_on;
|
||||
pcre2_code *pcre2_pattern;
|
||||
pcre2_match_data *pcre2_match_data;
|
||||
pcre2_compile_context *pcre2_compile_context;
|
||||
pcre2_match_context *pcre2_match_context;
|
||||
pcre2_jit_stack *pcre2_jit_stack;
|
||||
uint32_t pcre2_jit_on;
|
||||
kwset_t kws;
|
||||
unsigned fixed:1;
|
||||
unsigned is_fixed:1;
|
||||
unsigned ignore_case:1;
|
||||
unsigned word_regexp:1;
|
||||
};
|
||||
|
|
@ -176,6 +166,7 @@ struct grep_opt {
|
|||
int funcbody;
|
||||
int extended_regexp_option;
|
||||
int pattern_type_option;
|
||||
int ignore_locale;
|
||||
char colors[NR_GREP_COLORS][COLOR_MAXLEN];
|
||||
unsigned pre_context;
|
||||
unsigned post_context;
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@
|
|||
#include "commit-graph.h"
|
||||
#include "prio-queue.h"
|
||||
#include "hashmap.h"
|
||||
#include "utf8.h"
|
||||
|
||||
volatile show_early_output_fn_t show_early_output;
|
||||
|
||||
|
|
@ -2687,6 +2688,8 @@ int setup_revisions(int argc, const char **argv, struct rev_info *revs, struct s
|
|||
|
||||
grep_commit_pattern_type(GREP_PATTERN_TYPE_UNSPECIFIED,
|
||||
&revs->grep_filter);
|
||||
if (!is_encoding_utf8(get_log_output_encoding()))
|
||||
revs->grep_filter.ignore_locale = 1;
|
||||
compile_grep_patterns(&revs->grep_filter);
|
||||
|
||||
if (revs->reverse && revs->reflog_info)
|
||||
|
|
|
|||
|
|
@ -1,12 +1,15 @@
|
|||
#!/bin/sh
|
||||
|
||||
test_description='test log with i18n features'
|
||||
. ./test-lib.sh
|
||||
. ./lib-gettext.sh
|
||||
|
||||
# two forms of é
|
||||
utf8_e=$(printf '\303\251')
|
||||
latin1_e=$(printf '\351')
|
||||
|
||||
# invalid UTF-8
|
||||
invalid_e=$(printf '\303\50)') # ")" at end to close opening "("
|
||||
|
||||
test_expect_success 'create commits in different encodings' '
|
||||
test_tick &&
|
||||
cat >msg <<-EOF &&
|
||||
|
|
@ -48,9 +51,43 @@ test_expect_success !MINGW 'log --grep does not find non-reencoded values (utf8)
|
|||
test_must_be_empty actual
|
||||
'
|
||||
|
||||
test_expect_success 'log --grep does not find non-reencoded values (latin1)' '
|
||||
test_expect_success !MINGW 'log --grep does not find non-reencoded values (latin1)' '
|
||||
git log --encoding=ISO-8859-1 --format=%s --grep=$utf8_e >actual &&
|
||||
test_must_be_empty actual
|
||||
'
|
||||
|
||||
for engine in fixed basic extended perl
|
||||
do
|
||||
prereq=
|
||||
if test $engine = "perl"
|
||||
then
|
||||
prereq="PCRE"
|
||||
else
|
||||
prereq=""
|
||||
fi
|
||||
force_regex=
|
||||
if test $engine != "fixed"
|
||||
then
|
||||
force_regex=.*
|
||||
fi
|
||||
test_expect_success !MINGW,GETTEXT_LOCALE,$prereq "-c grep.patternType=$engine log --grep does not find non-reencoded values (latin1 + locale)" "
|
||||
cat >expect <<-\EOF &&
|
||||
latin1
|
||||
utf8
|
||||
EOF
|
||||
LC_ALL=\"$is_IS_locale\" git -c grep.patternType=$engine log --encoding=ISO-8859-1 --format=%s --grep=\"$force_regex$latin1_e\" >actual &&
|
||||
test_cmp expect actual
|
||||
"
|
||||
|
||||
test_expect_success !MINGW,GETTEXT_LOCALE,$prereq "-c grep.patternType=$engine log --grep does not find non-reencoded values (latin1 + locale)" "
|
||||
LC_ALL=\"$is_IS_locale\" git -c grep.patternType=$engine log --encoding=ISO-8859-1 --format=%s --grep=\"$force_regex$utf8_e\" >actual &&
|
||||
test_must_be_empty actual
|
||||
"
|
||||
|
||||
test_expect_success !MINGW,GETTEXT_LOCALE,$prereq "-c grep.patternType=$engine log --grep does not die on invalid UTF-8 value (latin1 + locale + invalid needle)" "
|
||||
LC_ALL=\"$is_IS_locale\" git -c grep.patternType=$engine log --encoding=ISO-8859-1 --format=%s --grep=\"$force_regex$invalid_e\" >actual &&
|
||||
test_must_be_empty actual
|
||||
"
|
||||
done
|
||||
|
||||
test_done
|
||||
|
|
|
|||
|
|
@ -53,4 +53,32 @@ test_expect_success REGEX_LOCALE 'pickaxe -i on non-ascii' '
|
|||
test_cmp expected actual
|
||||
'
|
||||
|
||||
test_expect_success GETTEXT_LOCALE,LIBPCRE2 'PCRE v2: setup invalid UTF-8 data' '
|
||||
printf "\\200\\n" >invalid-0x80 &&
|
||||
echo "ævar" >expected &&
|
||||
cat expected >>invalid-0x80 &&
|
||||
git add invalid-0x80
|
||||
'
|
||||
|
||||
test_expect_success GETTEXT_LOCALE,LIBPCRE2 'PCRE v2: grep ASCII from invalid UTF-8 data' '
|
||||
git grep -h "var" invalid-0x80 >actual &&
|
||||
test_cmp expected actual &&
|
||||
git grep -h "(*NO_JIT)var" invalid-0x80 >actual &&
|
||||
test_cmp expected actual
|
||||
'
|
||||
|
||||
test_expect_success GETTEXT_LOCALE,LIBPCRE2 'PCRE v2: grep non-ASCII from invalid UTF-8 data' '
|
||||
git grep -h "æ" invalid-0x80 >actual &&
|
||||
test_cmp expected actual &&
|
||||
git grep -h "(*NO_JIT)æ" invalid-0x80 &&
|
||||
test_cmp expected actual
|
||||
'
|
||||
|
||||
test_expect_success GETTEXT_LOCALE,LIBPCRE2 'PCRE v2: grep non-ASCII from invalid UTF-8 data with -i' '
|
||||
test_might_fail git grep -hi "Æ" invalid-0x80 >actual &&
|
||||
test_cmp expected actual &&
|
||||
test_must_fail git grep -hi "(*NO_JIT)Æ" invalid-0x80 &&
|
||||
test_cmp expected actual
|
||||
'
|
||||
|
||||
test_done
|
||||
|
|
|
|||
|
|
@ -4,41 +4,6 @@ test_description='git grep in binary files'
|
|||
|
||||
. ./test-lib.sh
|
||||
|
||||
nul_match () {
|
||||
matches=$1
|
||||
flags=$2
|
||||
pattern=$3
|
||||
pattern_human=$(echo "$pattern" | sed 's/Q/<NUL>/g')
|
||||
|
||||
if test "$matches" = 1
|
||||
then
|
||||
test_expect_success "git grep -f f $flags '$pattern_human' a" "
|
||||
printf '$pattern' | q_to_nul >f &&
|
||||
git grep -f f $flags a
|
||||
"
|
||||
elif test "$matches" = 0
|
||||
then
|
||||
test_expect_success "git grep -f f $flags '$pattern_human' a" "
|
||||
printf '$pattern' | q_to_nul >f &&
|
||||
test_must_fail git grep -f f $flags a
|
||||
"
|
||||
elif test "$matches" = T1
|
||||
then
|
||||
test_expect_failure "git grep -f f $flags '$pattern_human' a" "
|
||||
printf '$pattern' | q_to_nul >f &&
|
||||
git grep -f f $flags a
|
||||
"
|
||||
elif test "$matches" = T0
|
||||
then
|
||||
test_expect_failure "git grep -f f $flags '$pattern_human' a" "
|
||||
printf '$pattern' | q_to_nul >f &&
|
||||
test_must_fail git grep -f f $flags a
|
||||
"
|
||||
else
|
||||
test_expect_success "PANIC: Test framework error. Unknown matches value $matches" 'false'
|
||||
fi
|
||||
}
|
||||
|
||||
test_expect_success 'setup' "
|
||||
echo 'binaryQfileQm[*]cQ*æQð' | q_to_nul >a &&
|
||||
git add a &&
|
||||
|
|
@ -102,72 +67,6 @@ test_expect_failure 'git grep .fi a' '
|
|||
git grep .fi a
|
||||
'
|
||||
|
||||
nul_match 1 '-F' 'yQf'
|
||||
nul_match 0 '-F' 'yQx'
|
||||
nul_match 1 '-Fi' 'YQf'
|
||||
nul_match 0 '-Fi' 'YQx'
|
||||
nul_match 1 '' 'yQf'
|
||||
nul_match 0 '' 'yQx'
|
||||
nul_match 1 '' 'æQð'
|
||||
nul_match 1 '-F' 'eQm[*]c'
|
||||
nul_match 1 '-Fi' 'EQM[*]C'
|
||||
|
||||
# Regex patterns that would match but shouldn't with -F
|
||||
nul_match 0 '-F' 'yQ[f]'
|
||||
nul_match 0 '-F' '[y]Qf'
|
||||
nul_match 0 '-Fi' 'YQ[F]'
|
||||
nul_match 0 '-Fi' '[Y]QF'
|
||||
nul_match 0 '-F' 'æQ[ð]'
|
||||
nul_match 0 '-F' '[æ]Qð'
|
||||
nul_match 0 '-Fi' 'ÆQ[Ð]'
|
||||
nul_match 0 '-Fi' '[Æ]QÐ'
|
||||
|
||||
# kwset is disabled on -i & non-ASCII. No way to match non-ASCII \0
|
||||
# patterns case-insensitively.
|
||||
nul_match T1 '-i' 'ÆQÐ'
|
||||
|
||||
# \0 implicitly disables regexes. This is an undocumented internal
|
||||
# limitation.
|
||||
nul_match T1 '' 'yQ[f]'
|
||||
nul_match T1 '' '[y]Qf'
|
||||
nul_match T1 '-i' 'YQ[F]'
|
||||
nul_match T1 '-i' '[Y]Qf'
|
||||
nul_match T1 '' 'æQ[ð]'
|
||||
nul_match T1 '' '[æ]Qð'
|
||||
nul_match T1 '-i' 'ÆQ[Ð]'
|
||||
|
||||
# ... because of \0 implicitly disabling regexes regexes that
|
||||
# should/shouldn't match don't do the right thing.
|
||||
nul_match T1 '' 'eQm.*cQ'
|
||||
nul_match T1 '-i' 'EQM.*cQ'
|
||||
nul_match T0 '' 'eQm[*]c'
|
||||
nul_match T0 '-i' 'EQM[*]C'
|
||||
|
||||
# Due to the REG_STARTEND extension when kwset() is disabled on -i &
|
||||
# non-ASCII the string will be matched in its entirety, but the
|
||||
# pattern will be cut off at the first \0.
|
||||
nul_match 0 '-i' 'NOMATCHQð'
|
||||
nul_match T0 '-i' '[Æ]QNOMATCH'
|
||||
nul_match T0 '-i' '[æ]QNOMATCH'
|
||||
# Matches, but for the wrong reasons, just stops at [æ]
|
||||
nul_match 1 '-i' '[Æ]Qð'
|
||||
nul_match 1 '-i' '[æ]Qð'
|
||||
|
||||
# Ensure that the matcher doesn't regress to something that stops at
|
||||
# \0
|
||||
nul_match 0 '-F' 'yQ[f]'
|
||||
nul_match 0 '-Fi' 'YQ[F]'
|
||||
nul_match 0 '' 'yQNOMATCH'
|
||||
nul_match 0 '' 'QNOMATCH'
|
||||
nul_match 0 '-i' 'YQNOMATCH'
|
||||
nul_match 0 '-i' 'QNOMATCH'
|
||||
nul_match 0 '-F' 'æQ[ð]'
|
||||
nul_match 0 '-Fi' 'ÆQ[Ð]'
|
||||
nul_match 0 '' 'yQNÓMATCH'
|
||||
nul_match 0 '' 'QNÓMATCH'
|
||||
nul_match 0 '-i' 'YQNÓMATCH'
|
||||
nul_match 0 '-i' 'QNÓMATCH'
|
||||
|
||||
test_expect_success 'grep respects binary diff attribute' '
|
||||
echo text >t &&
|
||||
git add t &&
|
||||
|
|
@ -0,0 +1,127 @@
|
|||
#!/bin/sh
|
||||
|
||||
test_description='git grep with a binary pattern files'
|
||||
|
||||
. ./lib-gettext.sh
|
||||
|
||||
nul_match_internal () {
|
||||
matches=$1
|
||||
prereqs=$2
|
||||
lc_all=$3
|
||||
extra_flags=$4
|
||||
flags=$5
|
||||
pattern=$6
|
||||
pattern_human=$(echo "$pattern" | sed 's/Q/<NUL>/g')
|
||||
|
||||
if test "$matches" = 1
|
||||
then
|
||||
test_expect_success $prereqs "LC_ALL='$lc_all' git grep $extra_flags -f f $flags '$pattern_human' a" "
|
||||
printf '$pattern' | q_to_nul >f &&
|
||||
LC_ALL='$lc_all' git grep $extra_flags -f f $flags a
|
||||
"
|
||||
elif test "$matches" = 0
|
||||
then
|
||||
test_expect_success $prereqs "LC_ALL='$lc_all' git grep $extra_flags -f f $flags '$pattern_human' a" "
|
||||
>stderr &&
|
||||
printf '$pattern' | q_to_nul >f &&
|
||||
test_must_fail env LC_ALL=\"$lc_all\" git grep $extra_flags -f f $flags a 2>stderr &&
|
||||
test_i18ngrep ! 'This is only supported with -P under PCRE v2' stderr
|
||||
"
|
||||
elif test "$matches" = P
|
||||
then
|
||||
test_expect_success $prereqs "error, PCRE v2 only: LC_ALL='$lc_all' git grep -f f $flags '$pattern_human' a" "
|
||||
>stderr &&
|
||||
printf '$pattern' | q_to_nul >f &&
|
||||
test_must_fail env LC_ALL=\"$lc_all\" git grep -f f $flags a 2>stderr &&
|
||||
test_i18ngrep 'This is only supported with -P under PCRE v2' stderr
|
||||
"
|
||||
else
|
||||
test_expect_success "PANIC: Test framework error. Unknown matches value $matches" 'false'
|
||||
fi
|
||||
}
|
||||
|
||||
nul_match () {
|
||||
matches=$1
|
||||
matches_pcre2=$2
|
||||
matches_pcre2_locale=$3
|
||||
flags=$4
|
||||
pattern=$5
|
||||
pattern_human=$(echo "$pattern" | sed 's/Q/<NUL>/g')
|
||||
|
||||
nul_match_internal "$matches" "" "C" "" "$flags" "$pattern"
|
||||
nul_match_internal "$matches_pcre2" "LIBPCRE2" "C" "-P" "$flags" "$pattern"
|
||||
nul_match_internal "$matches_pcre2_locale" "LIBPCRE2,GETTEXT_LOCALE" "$is_IS_locale" "-P" "$flags" "$pattern"
|
||||
}
|
||||
|
||||
test_expect_success 'setup' "
|
||||
echo 'binaryQfileQm[*]cQ*æQð' | q_to_nul >a &&
|
||||
git add a &&
|
||||
git commit -m.
|
||||
"
|
||||
|
||||
# Simple fixed-string matching that can use kwset (no -i && non-ASCII)
|
||||
nul_match P P P '-F' 'yQf'
|
||||
nul_match P P P '-F' 'yQx'
|
||||
nul_match P P P '-Fi' 'YQf'
|
||||
nul_match P P P '-Fi' 'YQx'
|
||||
nul_match P P 1 '' 'yQf'
|
||||
nul_match P P 0 '' 'yQx'
|
||||
nul_match P P 1 '' 'æQð'
|
||||
nul_match P P P '-F' 'eQm[*]c'
|
||||
nul_match P P P '-Fi' 'EQM[*]C'
|
||||
|
||||
# Regex patterns that would match but shouldn't with -F
|
||||
nul_match P P P '-F' 'yQ[f]'
|
||||
nul_match P P P '-F' '[y]Qf'
|
||||
nul_match P P P '-Fi' 'YQ[F]'
|
||||
nul_match P P P '-Fi' '[Y]QF'
|
||||
nul_match P P P '-F' 'æQ[ð]'
|
||||
nul_match P P P '-F' '[æ]Qð'
|
||||
|
||||
# The -F kwset codepath can't handle -i && non-ASCII...
|
||||
nul_match P 1 1 '-i' '[æ]Qð'
|
||||
|
||||
# ...PCRE v2 only matches non-ASCII with -i casefolding under UTF-8
|
||||
# semantics
|
||||
nul_match P P P '-Fi' 'ÆQ[Ð]'
|
||||
nul_match P 0 1 '-i' 'ÆQ[Ð]'
|
||||
nul_match P 0 1 '-i' '[Æ]QÐ'
|
||||
nul_match P 0 1 '-i' '[Æ]Qð'
|
||||
nul_match P 0 1 '-i' 'ÆQÐ'
|
||||
|
||||
# \0 in regexes can only work with -P & PCRE v2
|
||||
nul_match P P 1 '' 'yQ[f]'
|
||||
nul_match P P 1 '' '[y]Qf'
|
||||
nul_match P P 1 '-i' 'YQ[F]'
|
||||
nul_match P P 1 '-i' '[Y]Qf'
|
||||
nul_match P P 1 '' 'æQ[ð]'
|
||||
nul_match P P 1 '' '[æ]Qð'
|
||||
nul_match P P 1 '-i' 'ÆQ[Ð]'
|
||||
nul_match P P 1 '' 'eQm.*cQ'
|
||||
nul_match P P 1 '-i' 'EQM.*cQ'
|
||||
nul_match P P 0 '' 'eQm[*]c'
|
||||
nul_match P P 0 '-i' 'EQM[*]C'
|
||||
|
||||
# Assert that we're using REG_STARTEND and the pattern doesn't match
|
||||
# just because it's cut off at the first \0.
|
||||
nul_match P P 0 '-i' 'NOMATCHQð'
|
||||
nul_match P P 0 '-i' '[Æ]QNOMATCH'
|
||||
nul_match P P 0 '-i' '[æ]QNOMATCH'
|
||||
|
||||
# Ensure that the matcher doesn't regress to something that stops at
|
||||
# \0
|
||||
nul_match P P P '-F' 'yQ[f]'
|
||||
nul_match P P P '-Fi' 'YQ[F]'
|
||||
nul_match P P 0 '' 'yQNOMATCH'
|
||||
nul_match P P 0 '' 'QNOMATCH'
|
||||
nul_match P P 0 '-i' 'YQNOMATCH'
|
||||
nul_match P P 0 '-i' 'QNOMATCH'
|
||||
nul_match P P P '-F' 'æQ[ð]'
|
||||
nul_match P P P '-Fi' 'ÆQ[Ð]'
|
||||
nul_match P P 1 '-i' 'ÆQ[Ð]'
|
||||
nul_match P P 0 '' 'yQNÓMATCH'
|
||||
nul_match P P 0 '' 'QNÓMATCH'
|
||||
nul_match P P 0 '-i' 'YQNÓMATCH'
|
||||
nul_match P P 0 '-i' 'QNÓMATCH'
|
||||
|
||||
test_done
|
||||
Loading…
Reference in New Issue