1
0
Fork 0
mirror of https://github.com/git/git.git synced 2024-04-26 12:25:10 +02:00
git/userdiff.c

545 lines
17 KiB
C
Raw Permalink Normal View History

#include "git-compat-util.h"
#include "config.h"
#include "userdiff.h"
#include "attr.h"
#include "strbuf.h"
#include "environment.h"
static struct userdiff_driver *drivers;
static int ndrivers;
static int drivers_alloc;
#define PATTERNS(lang, rx, wrx) { \
.name = lang, \
.binary = -1, \
.funcname = { \
.pattern = rx, \
.cflags = REG_EXTENDED, \
}, \
.word_regex = wrx "|[^[:space:]]|[\xc0-\xff][\x80-\xbf]+", \
userdiff: support regexec(3) with multi-byte support Since 1819ad327b (grep: fix multibyte regex handling under macOS, 2022-08-26) we use the system library for all regular expression matching on macOS, not just for git grep. It supports multi-byte strings and rejects invalid multi-byte characters. This broke all built-in userdiff word regexes in UTF-8 locales because they all include such invalid bytes in expressions that are intended to match multi-byte characters without explicit support for that from the regex engine. "|[^[:space:]]|[\xc0-\xff][\x80-\xbf]+" is added to all built-in word regexes to match a single non-space or multi-byte character. The \xNN characters are invalid if interpreted as UTF-8 because they have their high bit set, which indicates they are part of a multi-byte character, but they are surrounded by single-byte characters. Replace that expression with "|[^[:space:]]" if the regex engine supports multi-byte matching, as there is no need to have an explicit range for multi-byte characters then. Check for that capability at runtime, because it depends on the locale and thus on environment variables. Construct the full replacement expression at build time and just switch it in if necessary to avoid string manipulation and allocations at runtime. Additionally the word regex for tex contains the expression "[a-zA-Z0-9\x80-\xff]+" with a similarly invalid range. The best replacement with only valid characters that I can come up with is "([a-zA-Z0-9]|[^\x01-\x7f])+". Unlike the original it matches NUL characters, though. Assuming that tex files usually don't contain NUL this should be acceptable. Reported-by: D. Ben Knoble <ben.knoble@gmail.com> Reported-by: Eric Sunshine <sunshine@sunshineco.com> Helped-by: Junio C Hamano <gitster@pobox.com> Signed-off-by: René Scharfe <l.s.r@web.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-04-06 22:19:11 +02:00
.word_regex_multi_byte = wrx "|[^[:space:]]", \
}
#define IPATTERN(lang, rx, wrx) { \
.name = lang, \
.binary = -1, \
.funcname = { \
.pattern = rx, \
.cflags = REG_EXTENDED | REG_ICASE, \
}, \
.word_regex = wrx "|[^[:space:]]|[\xc0-\xff][\x80-\xbf]+", \
userdiff: support regexec(3) with multi-byte support Since 1819ad327b (grep: fix multibyte regex handling under macOS, 2022-08-26) we use the system library for all regular expression matching on macOS, not just for git grep. It supports multi-byte strings and rejects invalid multi-byte characters. This broke all built-in userdiff word regexes in UTF-8 locales because they all include such invalid bytes in expressions that are intended to match multi-byte characters without explicit support for that from the regex engine. "|[^[:space:]]|[\xc0-\xff][\x80-\xbf]+" is added to all built-in word regexes to match a single non-space or multi-byte character. The \xNN characters are invalid if interpreted as UTF-8 because they have their high bit set, which indicates they are part of a multi-byte character, but they are surrounded by single-byte characters. Replace that expression with "|[^[:space:]]" if the regex engine supports multi-byte matching, as there is no need to have an explicit range for multi-byte characters then. Check for that capability at runtime, because it depends on the locale and thus on environment variables. Construct the full replacement expression at build time and just switch it in if necessary to avoid string manipulation and allocations at runtime. Additionally the word regex for tex contains the expression "[a-zA-Z0-9\x80-\xff]+" with a similarly invalid range. The best replacement with only valid characters that I can come up with is "([a-zA-Z0-9]|[^\x01-\x7f])+". Unlike the original it matches NUL characters, though. Assuming that tex files usually don't contain NUL this should be acceptable. Reported-by: D. Ben Knoble <ben.knoble@gmail.com> Reported-by: Eric Sunshine <sunshine@sunshineco.com> Helped-by: Junio C Hamano <gitster@pobox.com> Signed-off-by: René Scharfe <l.s.r@web.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-04-06 22:19:11 +02:00
.word_regex_multi_byte = wrx "|[^[:space:]]", \
}
/*
* Built-in drivers for various languages, sorted by their names
* (except that the "default" is left at the end).
*
* When writing or updating patterns, assume that the contents these
* patterns are applied to are syntactically correct. The patterns
* can be simple without implementing all syntactical corner cases, as
* long as they are sufficiently permissive.
*/
static struct userdiff_driver builtin_drivers[] = {
IPATTERN("ada",
"!^(.*[ \t])?(is[ \t]+new|renames|is[ \t]+separate)([ \t].*)?$\n"
"!^[ \t]*with[ \t].*$\n"
"^[ \t]*((procedure|function)[ \t]+.*)$\n"
"^[ \t]*((package|protected|task)[ \t]+.*)$",
/* -- */
"[a-zA-Z][a-zA-Z0-9_]*"
"|[-+]?[0-9][0-9#_.aAbBcCdDeEfF]*([eE][+-]?[0-9_]+)?"
"|=>|\\.\\.|\\*\\*|:=|/=|>=|<=|<<|>>|<>"),
PATTERNS("bash",
/* Optional leading indentation */
"^[ \t]*"
/* Start of captured text */
"("
"("
/* POSIX identifier with mandatory parentheses */
"[a-zA-Z_][a-zA-Z0-9_]*[ \t]*\\([ \t]*\\))"
"|"
/* Bashism identifier with optional parentheses */
"(function[ \t]+[a-zA-Z_][a-zA-Z0-9_]*(([ \t]*\\([ \t]*\\))|([ \t]+))"
")"
/* Optional whitespace */
"[ \t]*"
/* Compound command starting with `{`, `(`, `((` or `[[` */
"(\\{|\\(\\(?|\\[\\[)"
/* End of captured text */
")",
/* -- */
/* Characters not in the default $IFS value */
"[^ \t]+"),
PATTERNS("bibtex",
"(@[a-zA-Z]{1,}[ \t]*\\{{0,1}[ \t]*[^ \t\"@',\\#}{~%]*).*$",
/* -- */
"[={}\"]|[^={}\" \t]+"),
PATTERNS("cpp",
/* Jump targets or access declarations */
"!^[ \t]*[A-Za-z_][A-Za-z_0-9]*:[[:space:]]*($|/[/*])\n"
/* functions/methods, variables, and compounds at top level */
"^((::[[:space:]]*)?[A-Za-z_].*)$",
/* -- */
/* identifiers and keywords */
"[a-zA-Z_][a-zA-Z0-9_]*"
/* decimal and octal integers as well as floatingpoint numbers */
"|[0-9][0-9.]*([Ee][-+]?[0-9]+)?[fFlLuU]*"
/* hexadecimal and binary integers */
"|0[xXbB][0-9a-fA-F]+[lLuU]*"
/* floatingpoint numbers that begin with a decimal point */
"|\\.[0-9][0-9]*([Ee][-+]?[0-9]+)?[fFlL]?"
"|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->\\*?|\\.\\*|<=>"),
PATTERNS("csharp",
/*
* Jump over reserved keywords which are illegal method names, but which
* can be followed by parentheses without special characters in between,
* making them look like methods.
*/
"!(^|[ \t]+)" /* Start of line or whitespace. */
"(do|while|for|foreach|if|else|new|default|return|switch|case|throw"
"|catch|using|lock|fixed)"
"([ \t(]+|$)\n" /* Whitespace, "(", or end of line. */
/*
* Methods/constructors:
* The strategy is to identify a minimum of two groups (any combination
* of keywords/type/name) before the opening parenthesis, and without
* final unexpected characters, normally only used in ordinary statements.
*/
"^[ \t]*" /* Remove leading whitespace. */
"(" /* Start chunk header capture. */
"(" /* First group. */
"[][[:alnum:]@_.]" /* Name. */
"(<[][[:alnum:]@_, \t<>]+>)?" /* Optional generic parameters. */
")+"
"([ \t]+" /* Subsequent groups, prepended with space. */
"([][[:alnum:]@_.](<[][[:alnum:]@_, \t<>]+>)?)+"
")+"
"[ \t]*" /* Optional space before parameters start. */
"\\(" /* Start of method parameters. */
"[^;]*" /* Allow complex parameters, but exclude statements (;). */
")$\n" /* Close chunk header capture. */
/*
* Properties:
* As with methods, expect a minimum of two groups. But, more trivial than
* methods, the vast majority of properties long enough to be worth
* showing a chunk header for don't include "=:;,()" on the line they are
* defined, since they don't have a parameter list.
*/
"^[ \t]*("
"([][[:alnum:]@_.](<[][[:alnum:]@_, \t<>]+>)?)+"
"([ \t]+"
"([][[:alnum:]@_.](<[][[:alnum:]@_, \t<>]+>)?)+"
")+" /* Up to here, same as methods regex. */
"[^;=:,()]*" /* Compared to methods, no parameter list allowed. */
")$\n"
/* Type definitions */
"^[ \t]*(((static|public|internal|private|protected|new|unsafe|sealed|abstract|partial)[ \t]+)*(class|enum|interface|struct|record)[ \t]+.*)$\n"
/* Namespace */
"^[ \t]*(namespace[ \t]+.*)$",
/* -- */
"[a-zA-Z_][a-zA-Z0-9_]*"
"|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?"
"|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"),
IPATTERN("css",
"![:;][[:space:]]*$\n"
"^[:[@.#]?[_a-z0-9].*$",
/* -- */
/*
* This regex comes from W3C CSS specs. Should theoretically also
* allow ISO 10646 characters U+00A0 and higher,
* but they are not handled in this regex.
*/
"-?[_a-zA-Z][-_a-zA-Z0-9]*" /* identifiers */
"|-?[0-9]+|\\#[0-9a-fA-F]+" /* numbers */
),
PATTERNS("dts",
"!;\n"
"!=\n"
/* lines beginning with a word optionally preceded by '&' or the root */
"^[ \t]*((/[ \t]*\\{|&?[a-zA-Z_]).*)",
/* -- */
/* Property names and math operators */
"[a-zA-Z0-9,._+?#-]+"
"|[-+*/%&^|!~]|>>|<<|&&|\\|\\|"),
PATTERNS("elixir",
"^[ \t]*((def(macro|module|impl|protocol|p)?|test)[ \t].*)$",
/* -- */
/* Atoms, names, and module attributes */
"[@:]?[a-zA-Z0-9@_?!]+"
/* Numbers with specific base */
"|[-+]?0[xob][0-9a-fA-F]+"
/* Numbers */
"|[-+]?[0-9][0-9_.]*([eE][-+]?[0-9_]+)?"
/* Operators and atoms that represent them */
"|:?(\\+\\+|--|\\.\\.|~~~|<>|\\^\\^\\^|<?\\|>|<<<?|>?>>|<<?~|~>?>|<~>|<=|>=|===?|!==?|=~|&&&?|\\|\\|\\|?|=>|<-|\\\\\\\\|->)"
/* Not real operators, but should be grouped */
"|:?%[A-Za-z0-9_.]\\{\\}?"),
IPATTERN("fortran",
/* Don't match comment lines */
"!^([C*]|[ \t]*!)\n"
/* Don't match 'module procedure' lines */
"!^[ \t]*MODULE[ \t]+PROCEDURE[ \t]\n"
/* Program, module, block data */
"^[ \t]*((END[ \t]+)?(PROGRAM|MODULE|BLOCK[ \t]+DATA"
/* Subroutines and functions */
"|([^!'\" \t]+[ \t]+)*(SUBROUTINE|FUNCTION))[ \t]+[A-Z].*)$",
/* -- */
"[a-zA-Z][a-zA-Z0-9_]*"
"|\\.([Ee][Qq]|[Nn][Ee]|[Gg][TtEe]|[Ll][TtEe]|[Tt][Rr][Uu][Ee]|[Ff][Aa][Ll][Ss][Ee]|[Aa][Nn][Dd]|[Oo][Rr]|[Nn]?[Ee][Qq][Vv]|[Nn][Oo][Tt])\\."
/* numbers and format statements like 2E14.4, or ES12.6, 9X.
* Don't worry about format statements without leading digits since
* they would have been matched above as a variable anyway. */
"|[-+]?[0-9.]+([AaIiDdEeFfLlTtXx][Ss]?[-+]?[0-9.]*)?(_[a-zA-Z0-9][a-zA-Z0-9_]*)?"
"|//|\\*\\*|::|[/<>=]="),
IPATTERN("fountain",
"^((\\.[^.]|(int|ext|est|int\\.?/ext|i/e)[. ]).*)$",
/* -- */
"[^ \t-]+"),
PATTERNS("golang",
/* Functions */
"^[ \t]*(func[ \t]*.*(\\{[ \t]*)?)\n"
/* Structs and interfaces */
"^[ \t]*(type[ \t].*(struct|interface)[ \t]*(\\{[ \t]*)?)",
/* -- */
"[a-zA-Z_][a-zA-Z0-9_]*"
"|[-+0-9.eE]+i?|0[xX]?[0-9a-fA-F]+i?"
"|[-+*/<>%&^|=!:]=|--|\\+\\+|<<=?|>>=?|&\\^=?|&&|\\|\\||<-|\\.{3}"),
PATTERNS("html",
"^[ \t]*(<[Hh][1-6]([ \t].*)?>.*)$",
/* -- */
"[^<>= \t]+"),
PATTERNS("java",
"!^[ \t]*(catch|do|for|if|instanceof|new|return|switch|throw|while)\n"
/* Class, enum, interface, and record declarations */
"^[ \t]*(([a-z-]+[ \t]+)*(class|enum|interface|record)[ \t]+.*)$\n"
/* Method definitions; note that constructor signatures are not */
/* matched because they are indistinguishable from method calls. */
"^[ \t]*(([A-Za-z_<>&][][?&<>.,A-Za-z_0-9]*[ \t]+)+[A-Za-z_][A-Za-z_0-9]*[ \t]*\\([^;]*)$",
/* -- */
"[a-zA-Z_][a-zA-Z0-9_]*"
"|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?"
"|[-+*/<>%&^|=!]="
"|--|\\+\\+|<<=?|>>>?=?|&&|\\|\\|"),
PATTERNS("kotlin",
"^[ \t]*(([a-z]+[ \t]+)*(fun|class|interface)[ \t]+.*)$",
/* -- */
"[a-zA-Z_][a-zA-Z0-9_]*"
/* hexadecimal and binary numbers */
"|0[xXbB][0-9a-fA-F_]+[lLuU]*"
/* integers and floats */
"|[0-9][0-9_]*([.][0-9_]*)?([Ee][-+]?[0-9]+)?[fFlLuU]*"
/* floating point numbers beginning with decimal point */
"|[.][0-9][0-9_]*([Ee][-+]?[0-9]+)?[fFlLuU]?"
/* unary and binary operators */
"|[-+*/<>%&^|=!]==?|--|\\+\\+|<<=|>>=|&&|\\|\\||->|\\.\\*|!!|[?:.][.:]"),
PATTERNS("markdown",
"^ {0,3}#{1,6}[ \t].*",
/* -- */
"[^<>= \t]+"),
PATTERNS("matlab",
/*
* Octave pattern is mostly the same as matlab, except that '%%%' and
* '##' can also be used to begin code sections, in addition to '%%'
* that is understood by both.
*/
"^[[:space:]]*((classdef|function)[[:space:]].*)$|^(%%%?|##)[[:space:]].*$",
/* -- */
"[a-zA-Z_][a-zA-Z0-9_]*|[-+0-9.e]+|[=~<>]=|\\.[*/\\^']|\\|\\||&&"),
PATTERNS("objc",
/* Negate C statements that can look like functions */
"!^[ \t]*(do|for|if|else|return|switch|while)\n"
/* Objective-C methods */
"^[ \t]*([-+][ \t]*\\([ \t]*[A-Za-z_][A-Za-z_0-9* \t]*\\)[ \t]*[A-Za-z_].*)$\n"
/* C functions */
"^[ \t]*(([A-Za-z_][A-Za-z_0-9]*[ \t]+)+[A-Za-z_][A-Za-z_0-9]*[ \t]*\\([^;]*)$\n"
/* Objective-C class/protocol definitions */
"^(@(implementation|interface|protocol)[ \t].*)$",
/* -- */
"[a-zA-Z_][a-zA-Z0-9_]*"
"|[-+0-9.e]+[fFlL]?|0[xXbB]?[0-9a-fA-F]+[lL]?"
"|[-+*/<>%&^|=!]=|--|\\+\\+|<<=?|>>=?|&&|\\|\\||::|->"),
PATTERNS("pascal",
"^(((class[ \t]+)?(procedure|function)|constructor|destructor|interface"
"|implementation|initialization|finalization)[ \t]*.*)$\n"
"^(.*=[ \t]*(class|record).*)$",
/* -- */
"[a-zA-Z_][a-zA-Z0-9_]*"
"|[-+0-9.e]+|0[xXbB]?[0-9a-fA-F]+"
"|<>|<=|>=|:=|\\.\\."),
PATTERNS("perl",
"^package .*\n"
"^sub [[:alnum:]_':]+[ \t]*"
"(\\([^)]*\\)[ \t]*)?" /* prototype */
/*
* Attributes. A regex can't count nested parentheses,
* so just slurp up whatever we see, taking care not
* to accept lines like "sub foo; # defined elsewhere".
*
* An attribute could contain a semicolon, but at that
* point it seems reasonable enough to give up.
*/
"(:[^;#]*)?"
"(\\{[ \t]*)?" /* brace can come here or on the next line */
"(#.*)?$\n" /* comment */
"^(BEGIN|END|INIT|CHECK|UNITCHECK|AUTOLOAD|DESTROY)[ \t]*"
"(\\{[ \t]*)?" /* brace can come here or on the next line */
"(#.*)?$\n"
"^=head[0-9] .*", /* POD */
/* -- */
"[[:alpha:]_'][[:alnum:]_']*"
"|0[xb]?[0-9a-fA-F_]*"
/* taking care not to interpret 3..5 as (3.)(.5) */
"|[0-9a-fA-F_]+(\\.[0-9a-fA-F_]+)?([eE][-+]?[0-9_]+)?"
"|=>|-[rwxoRWXOezsfdlpSugkbctTBMAC>]|~~|::"
"|&&=|\\|\\|=|//=|\\*\\*="
"|&&|\\|\\||//|\\+\\+|--|\\*\\*|\\.\\.\\.?"
"|[-+*/%.^&<>=!|]="
"|=~|!~"
"|<<|<>|<=>|>>"),
PATTERNS("php",
"^[\t ]*(((public|protected|private|static|abstract|final)[\t ]+)*function.*)$\n"
"^[\t ]*((((final|abstract)[\t ]+)?class|enum|interface|trait).*)$",
/* -- */
"[a-zA-Z_][a-zA-Z0-9_]*"
"|[-+0-9.e]+|0[xXbB]?[0-9a-fA-F]+"
"|[-+*/<>%&^|=!.]=|--|\\+\\+|<<=?|>>=?|===|&&|\\|\\||::|->"),
PATTERNS("python",
"^[ \t]*((class|(async[ \t]+)?def)[ \t].*)$",
/* -- */
"[a-zA-Z_][a-zA-Z0-9_]*"
"|[-+0-9.e]+[jJlL]?|0[xX]?[0-9a-fA-F]+[lL]?"
"|[-+*/<>%&^|=!]=|//=?|<<=?|>>=?|\\*\\*=?"),
/* -- */
PATTERNS("ruby",
"^[ \t]*((class|module|def)[ \t].*)$",
/* -- */
"(@|@@|\\$)?[a-zA-Z_][a-zA-Z0-9_]*"
"|[-+0-9.e]+|0[xXbB]?[0-9a-fA-F]+|\\?(\\\\C-)?(\\\\M-)?."
"|//=?|[-+*/<>%&^|=!]=|<<=?|>>=?|===|\\.{1,3}|::|[!=]~"),
PATTERNS("rust",
"^[\t ]*((pub(\\([^\\)]+\\))?[\t ]+)?((async|const|unsafe|extern([\t ]+\"[^\"]+\"))[\t ]+)?(struct|enum|union|mod|trait|fn|impl|macro_rules!)[< \t]+[^;]*)$",
/* -- */
"[a-zA-Z_][a-zA-Z0-9_]*"
"|[0-9][0-9_a-fA-Fiosuxz]*(\\.([0-9]*[eE][+-]?)?[0-9_fF]*)?"
"|[-+*\\/<>%&^|=!:]=|<<=?|>>=?|&&|\\|\\||->|=>|\\.{2}=|\\.{3}|::"),
PATTERNS("scheme",
"^[\t ]*(\\(((define|def(struct|syntax|class|method|rules|record|proto|alias)?)[-*/ \t]|(library|module|struct|class)[*+ \t]).*)$",
/*
* R7RS valid identifiers include any sequence enclosed
* within vertical lines having no backslashes
*/
"\\|([^\\\\]*)\\|"
/* All other words should be delimited by spaces or parentheses */
"|([^][)(}{[ \t])+"),
PATTERNS("tex", "^(\\\\((sub)*section|chapter|part)\\*{0,1}\\{.*)$",
userdiff: support regexec(3) with multi-byte support Since 1819ad327b (grep: fix multibyte regex handling under macOS, 2022-08-26) we use the system library for all regular expression matching on macOS, not just for git grep. It supports multi-byte strings and rejects invalid multi-byte characters. This broke all built-in userdiff word regexes in UTF-8 locales because they all include such invalid bytes in expressions that are intended to match multi-byte characters without explicit support for that from the regex engine. "|[^[:space:]]|[\xc0-\xff][\x80-\xbf]+" is added to all built-in word regexes to match a single non-space or multi-byte character. The \xNN characters are invalid if interpreted as UTF-8 because they have their high bit set, which indicates they are part of a multi-byte character, but they are surrounded by single-byte characters. Replace that expression with "|[^[:space:]]" if the regex engine supports multi-byte matching, as there is no need to have an explicit range for multi-byte characters then. Check for that capability at runtime, because it depends on the locale and thus on environment variables. Construct the full replacement expression at build time and just switch it in if necessary to avoid string manipulation and allocations at runtime. Additionally the word regex for tex contains the expression "[a-zA-Z0-9\x80-\xff]+" with a similarly invalid range. The best replacement with only valid characters that I can come up with is "([a-zA-Z0-9]|[^\x01-\x7f])+". Unlike the original it matches NUL characters, though. Assuming that tex files usually don't contain NUL this should be acceptable. Reported-by: D. Ben Knoble <ben.knoble@gmail.com> Reported-by: Eric Sunshine <sunshine@sunshineco.com> Helped-by: Junio C Hamano <gitster@pobox.com> Signed-off-by: René Scharfe <l.s.r@web.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-04-06 22:19:11 +02:00
"\\\\[a-zA-Z@]+|\\\\.|([a-zA-Z0-9]|[^\x01-\x7f])+"),
{ "default", NULL, NULL, -1, { NULL, 0 } },
};
#undef PATTERNS
#undef IPATTERN
static struct userdiff_driver driver_true = {
.name = "diff=true",
.binary = 0,
};
static struct userdiff_driver driver_false = {
.name = "!diff",
.binary = 1,
};
struct find_by_namelen_data {
const char *name;
size_t len;
struct userdiff_driver *driver;
};
static int userdiff_find_by_namelen_cb(struct userdiff_driver *driver,
enum userdiff_driver_type type UNUSED,
void *priv)
{
struct find_by_namelen_data *cb_data = priv;
if (!xstrncmpz(driver->name, cb_data->name, cb_data->len)) {
cb_data->driver = driver;
return 1; /* tell the caller to stop iterating */
}
return 0;
}
userdiff: support regexec(3) with multi-byte support Since 1819ad327b (grep: fix multibyte regex handling under macOS, 2022-08-26) we use the system library for all regular expression matching on macOS, not just for git grep. It supports multi-byte strings and rejects invalid multi-byte characters. This broke all built-in userdiff word regexes in UTF-8 locales because they all include such invalid bytes in expressions that are intended to match multi-byte characters without explicit support for that from the regex engine. "|[^[:space:]]|[\xc0-\xff][\x80-\xbf]+" is added to all built-in word regexes to match a single non-space or multi-byte character. The \xNN characters are invalid if interpreted as UTF-8 because they have their high bit set, which indicates they are part of a multi-byte character, but they are surrounded by single-byte characters. Replace that expression with "|[^[:space:]]" if the regex engine supports multi-byte matching, as there is no need to have an explicit range for multi-byte characters then. Check for that capability at runtime, because it depends on the locale and thus on environment variables. Construct the full replacement expression at build time and just switch it in if necessary to avoid string manipulation and allocations at runtime. Additionally the word regex for tex contains the expression "[a-zA-Z0-9\x80-\xff]+" with a similarly invalid range. The best replacement with only valid characters that I can come up with is "([a-zA-Z0-9]|[^\x01-\x7f])+". Unlike the original it matches NUL characters, though. Assuming that tex files usually don't contain NUL this should be acceptable. Reported-by: D. Ben Knoble <ben.knoble@gmail.com> Reported-by: Eric Sunshine <sunshine@sunshineco.com> Helped-by: Junio C Hamano <gitster@pobox.com> Signed-off-by: René Scharfe <l.s.r@web.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-04-06 22:19:11 +02:00
static int regexec_supports_multi_byte_chars(void)
{
static const char not_space[] = "[^[:space:]]";
static const char utf8_multi_byte_char[] = "\xc2\xa3";
regex_t re;
regmatch_t match;
static int result = -1;
if (result != -1)
return result;
if (regcomp(&re, not_space, REG_EXTENDED))
BUG("invalid regular expression: %s", not_space);
result = !regexec(&re, utf8_multi_byte_char, 1, &match, 0) &&
match.rm_so == 0 &&
match.rm_eo == strlen(utf8_multi_byte_char);
regfree(&re);
return result;
}
static struct userdiff_driver *userdiff_find_by_namelen(const char *name, size_t len)
{
struct find_by_namelen_data udcbdata = {
.name = name,
.len = len,
};
for_each_userdiff_driver(userdiff_find_by_namelen_cb, &udcbdata);
return udcbdata.driver;
}
static int parse_funcname(struct userdiff_funcname *f, const char *k,
const char *v, int cflags)
{
if (git_config_string(&f->pattern, k, v) < 0)
return -1;
f->cflags = cflags;
drop odd return value semantics from userdiff_config When the userdiff_config function was introduced in be58e70 (diff: unify external diff and funcname parsing code, 2008-10-05), it used a return value convention unlike any other config callback. Like other callbacks, it used "-1" to signal error. But it returned "1" to indicate that it found something, and "0" otherwise; other callbacks simply returned "0" to indicate that no error occurred. This distinction was necessary at the time, because the userdiff namespace overlapped slightly with the color configuration namespace. So "diff.color.foo" could mean "the 'foo' slot of diff coloring" or "the 'foo' component of the "color" userdiff driver". Because the color-parsing code would die on an unknown color slot, we needed the userdiff code to indicate that it had matched the variable, letting us bypass the color-parsing code entirely. Later, in 8b8e862 (ignore unknown color configuration, 2009-12-12), the color-parsing code learned to silently ignore unknown slots. This means we no longer need to protect userdiff-matched variables from reaching the color-parsing code. We can therefore change the userdiff_config calling convention to a more normal one. This drops some code from each caller, which is nice. But more importantly, it reduces the cognitive load for readers who may wonder why userdiff_config is unlike every other config callback. There's no need to add a new test confirming that this works; t4020 already contains a test that sets diff.color.external. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2012-02-07 19:23:02 +01:00
return 0;
}
diff: introduce diff.<driver>.binary The "diff" gitattribute is somewhat overloaded right now. It can say one of three things: 1. this file is definitely binary, or definitely not (i.e., diff or !diff) 2. this file should use an external diff engine (i.e., diff=foo, diff.foo.command = custom-script) 3. this file should use particular funcname patterns (i.e., diff=foo, diff.foo.(x?)funcname = some-regex) Most of the time, there is no conflict between these uses, since using one implies that the other is irrelevant (e.g., an external diff engine will decide for itself whether the file is binary). However, there is at least one conflicting situation: there is no way to say "use the regular rules to determine whether this file is binary, but if we do diff it textually, use this funcname pattern." That is, currently setting diff=foo indicates that the file is definitely text. This patch introduces a "binary" config option for a diff driver, so that one can explicitly set diff.foo.binary. We default this value to "don't know". That is, setting a diff attribute to "foo" and using "diff.foo.funcname" will have no effect on the binaryness of a file. To get the current behavior, one can set diff.foo.binary to true. This patch also has one additional advantage: it cleans up the interface to the userdiff code a bit. Before, calling code had to know more about whether attributes were false, true, or unset to determine binaryness. Now that binaryness is a property of a driver, we can represent these situations just by passing back a driver struct. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
2008-10-05 23:43:36 +02:00
static int parse_tristate(int *b, const char *k, const char *v)
{
if (v && !strcasecmp(v, "auto"))
*b = -1;
else
*b = git_config_bool(k, v);
drop odd return value semantics from userdiff_config When the userdiff_config function was introduced in be58e70 (diff: unify external diff and funcname parsing code, 2008-10-05), it used a return value convention unlike any other config callback. Like other callbacks, it used "-1" to signal error. But it returned "1" to indicate that it found something, and "0" otherwise; other callbacks simply returned "0" to indicate that no error occurred. This distinction was necessary at the time, because the userdiff namespace overlapped slightly with the color configuration namespace. So "diff.color.foo" could mean "the 'foo' slot of diff coloring" or "the 'foo' component of the "color" userdiff driver". Because the color-parsing code would die on an unknown color slot, we needed the userdiff code to indicate that it had matched the variable, letting us bypass the color-parsing code entirely. Later, in 8b8e862 (ignore unknown color configuration, 2009-12-12), the color-parsing code learned to silently ignore unknown slots. This means we no longer need to protect userdiff-matched variables from reaching the color-parsing code. We can therefore change the userdiff_config calling convention to a more normal one. This drops some code from each caller, which is nice. But more importantly, it reduces the cognitive load for readers who may wonder why userdiff_config is unlike every other config callback. There's no need to add a new test confirming that this works; t4020 already contains a test that sets diff.color.external. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2012-02-07 19:23:02 +01:00
return 0;
diff: introduce diff.<driver>.binary The "diff" gitattribute is somewhat overloaded right now. It can say one of three things: 1. this file is definitely binary, or definitely not (i.e., diff or !diff) 2. this file should use an external diff engine (i.e., diff=foo, diff.foo.command = custom-script) 3. this file should use particular funcname patterns (i.e., diff=foo, diff.foo.(x?)funcname = some-regex) Most of the time, there is no conflict between these uses, since using one implies that the other is irrelevant (e.g., an external diff engine will decide for itself whether the file is binary). However, there is at least one conflicting situation: there is no way to say "use the regular rules to determine whether this file is binary, but if we do diff it textually, use this funcname pattern." That is, currently setting diff=foo indicates that the file is definitely text. This patch introduces a "binary" config option for a diff driver, so that one can explicitly set diff.foo.binary. We default this value to "don't know". That is, setting a diff attribute to "foo" and using "diff.foo.funcname" will have no effect on the binaryness of a file. To get the current behavior, one can set diff.foo.binary to true. This patch also has one additional advantage: it cleans up the interface to the userdiff code a bit. Before, calling code had to know more about whether attributes were false, true, or unset to determine binaryness. Now that binaryness is a property of a driver, we can represent these situations just by passing back a driver struct. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
2008-10-05 23:43:36 +02:00
}
static int parse_bool(int *b, const char *k, const char *v)
{
*b = git_config_bool(k, v);
drop odd return value semantics from userdiff_config When the userdiff_config function was introduced in be58e70 (diff: unify external diff and funcname parsing code, 2008-10-05), it used a return value convention unlike any other config callback. Like other callbacks, it used "-1" to signal error. But it returned "1" to indicate that it found something, and "0" otherwise; other callbacks simply returned "0" to indicate that no error occurred. This distinction was necessary at the time, because the userdiff namespace overlapped slightly with the color configuration namespace. So "diff.color.foo" could mean "the 'foo' slot of diff coloring" or "the 'foo' component of the "color" userdiff driver". Because the color-parsing code would die on an unknown color slot, we needed the userdiff code to indicate that it had matched the variable, letting us bypass the color-parsing code entirely. Later, in 8b8e862 (ignore unknown color configuration, 2009-12-12), the color-parsing code learned to silently ignore unknown slots. This means we no longer need to protect userdiff-matched variables from reaching the color-parsing code. We can therefore change the userdiff_config calling convention to a more normal one. This drops some code from each caller, which is nice. But more importantly, it reduces the cognitive load for readers who may wonder why userdiff_config is unlike every other config callback. There's no need to add a new test confirming that this works; t4020 already contains a test that sets diff.color.external. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2012-02-07 19:23:02 +01:00
return 0;
}
int userdiff_config(const char *k, const char *v)
{
struct userdiff_driver *drv;
const char *name, *type;
size_t namelen;
if (parse_config_key(k, "diff", &name, &namelen, &type) || !name)
return 0;
drv = userdiff_find_by_namelen(name, namelen);
if (!drv) {
ALLOC_GROW(drivers, ndrivers+1, drivers_alloc);
drv = &drivers[ndrivers++];
memset(drv, 0, sizeof(*drv));
drv->name = xmemdupz(name, namelen);
drv->binary = -1;
}
if (!strcmp(type, "funcname"))
return parse_funcname(&drv->funcname, k, v, 0);
if (!strcmp(type, "xfuncname"))
return parse_funcname(&drv->funcname, k, v, REG_EXTENDED);
if (!strcmp(type, "binary"))
diff: introduce diff.<driver>.binary The "diff" gitattribute is somewhat overloaded right now. It can say one of three things: 1. this file is definitely binary, or definitely not (i.e., diff or !diff) 2. this file should use an external diff engine (i.e., diff=foo, diff.foo.command = custom-script) 3. this file should use particular funcname patterns (i.e., diff=foo, diff.foo.(x?)funcname = some-regex) Most of the time, there is no conflict between these uses, since using one implies that the other is irrelevant (e.g., an external diff engine will decide for itself whether the file is binary). However, there is at least one conflicting situation: there is no way to say "use the regular rules to determine whether this file is binary, but if we do diff it textually, use this funcname pattern." That is, currently setting diff=foo indicates that the file is definitely text. This patch introduces a "binary" config option for a diff driver, so that one can explicitly set diff.foo.binary. We default this value to "don't know". That is, setting a diff attribute to "foo" and using "diff.foo.funcname" will have no effect on the binaryness of a file. To get the current behavior, one can set diff.foo.binary to true. This patch also has one additional advantage: it cleans up the interface to the userdiff code a bit. Before, calling code had to know more about whether attributes were false, true, or unset to determine binaryness. Now that binaryness is a property of a driver, we can represent these situations just by passing back a driver struct. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
2008-10-05 23:43:36 +02:00
return parse_tristate(&drv->binary, k, v);
if (!strcmp(type, "command"))
drop odd return value semantics from userdiff_config When the userdiff_config function was introduced in be58e70 (diff: unify external diff and funcname parsing code, 2008-10-05), it used a return value convention unlike any other config callback. Like other callbacks, it used "-1" to signal error. But it returned "1" to indicate that it found something, and "0" otherwise; other callbacks simply returned "0" to indicate that no error occurred. This distinction was necessary at the time, because the userdiff namespace overlapped slightly with the color configuration namespace. So "diff.color.foo" could mean "the 'foo' slot of diff coloring" or "the 'foo' component of the "color" userdiff driver". Because the color-parsing code would die on an unknown color slot, we needed the userdiff code to indicate that it had matched the variable, letting us bypass the color-parsing code entirely. Later, in 8b8e862 (ignore unknown color configuration, 2009-12-12), the color-parsing code learned to silently ignore unknown slots. This means we no longer need to protect userdiff-matched variables from reaching the color-parsing code. We can therefore change the userdiff_config calling convention to a more normal one. This drops some code from each caller, which is nice. But more importantly, it reduces the cognitive load for readers who may wonder why userdiff_config is unlike every other config callback. There's no need to add a new test confirming that this works; t4020 already contains a test that sets diff.color.external. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2012-02-07 19:23:02 +01:00
return git_config_string(&drv->external, k, v);
if (!strcmp(type, "textconv"))
drop odd return value semantics from userdiff_config When the userdiff_config function was introduced in be58e70 (diff: unify external diff and funcname parsing code, 2008-10-05), it used a return value convention unlike any other config callback. Like other callbacks, it used "-1" to signal error. But it returned "1" to indicate that it found something, and "0" otherwise; other callbacks simply returned "0" to indicate that no error occurred. This distinction was necessary at the time, because the userdiff namespace overlapped slightly with the color configuration namespace. So "diff.color.foo" could mean "the 'foo' slot of diff coloring" or "the 'foo' component of the "color" userdiff driver". Because the color-parsing code would die on an unknown color slot, we needed the userdiff code to indicate that it had matched the variable, letting us bypass the color-parsing code entirely. Later, in 8b8e862 (ignore unknown color configuration, 2009-12-12), the color-parsing code learned to silently ignore unknown slots. This means we no longer need to protect userdiff-matched variables from reaching the color-parsing code. We can therefore change the userdiff_config calling convention to a more normal one. This drops some code from each caller, which is nice. But more importantly, it reduces the cognitive load for readers who may wonder why userdiff_config is unlike every other config callback. There's no need to add a new test confirming that this works; t4020 already contains a test that sets diff.color.external. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2012-02-07 19:23:02 +01:00
return git_config_string(&drv->textconv, k, v);
if (!strcmp(type, "cachetextconv"))
return parse_bool(&drv->textconv_want_cache, k, v);
if (!strcmp(type, "wordregex"))
drop odd return value semantics from userdiff_config When the userdiff_config function was introduced in be58e70 (diff: unify external diff and funcname parsing code, 2008-10-05), it used a return value convention unlike any other config callback. Like other callbacks, it used "-1" to signal error. But it returned "1" to indicate that it found something, and "0" otherwise; other callbacks simply returned "0" to indicate that no error occurred. This distinction was necessary at the time, because the userdiff namespace overlapped slightly with the color configuration namespace. So "diff.color.foo" could mean "the 'foo' slot of diff coloring" or "the 'foo' component of the "color" userdiff driver". Because the color-parsing code would die on an unknown color slot, we needed the userdiff code to indicate that it had matched the variable, letting us bypass the color-parsing code entirely. Later, in 8b8e862 (ignore unknown color configuration, 2009-12-12), the color-parsing code learned to silently ignore unknown slots. This means we no longer need to protect userdiff-matched variables from reaching the color-parsing code. We can therefore change the userdiff_config calling convention to a more normal one. This drops some code from each caller, which is nice. But more importantly, it reduces the cognitive load for readers who may wonder why userdiff_config is unlike every other config callback. There's no need to add a new test confirming that this works; t4020 already contains a test that sets diff.color.external. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2012-02-07 19:23:02 +01:00
return git_config_string(&drv->word_regex, k, v);
if (!strcmp(type, "algorithm"))
return git_config_string(&drv->algorithm, k, v);
return 0;
}
struct userdiff_driver *userdiff_find_by_name(const char *name)
{
int len = strlen(name);
userdiff: support regexec(3) with multi-byte support Since 1819ad327b (grep: fix multibyte regex handling under macOS, 2022-08-26) we use the system library for all regular expression matching on macOS, not just for git grep. It supports multi-byte strings and rejects invalid multi-byte characters. This broke all built-in userdiff word regexes in UTF-8 locales because they all include such invalid bytes in expressions that are intended to match multi-byte characters without explicit support for that from the regex engine. "|[^[:space:]]|[\xc0-\xff][\x80-\xbf]+" is added to all built-in word regexes to match a single non-space or multi-byte character. The \xNN characters are invalid if interpreted as UTF-8 because they have their high bit set, which indicates they are part of a multi-byte character, but they are surrounded by single-byte characters. Replace that expression with "|[^[:space:]]" if the regex engine supports multi-byte matching, as there is no need to have an explicit range for multi-byte characters then. Check for that capability at runtime, because it depends on the locale and thus on environment variables. Construct the full replacement expression at build time and just switch it in if necessary to avoid string manipulation and allocations at runtime. Additionally the word regex for tex contains the expression "[a-zA-Z0-9\x80-\xff]+" with a similarly invalid range. The best replacement with only valid characters that I can come up with is "([a-zA-Z0-9]|[^\x01-\x7f])+". Unlike the original it matches NUL characters, though. Assuming that tex files usually don't contain NUL this should be acceptable. Reported-by: D. Ben Knoble <ben.knoble@gmail.com> Reported-by: Eric Sunshine <sunshine@sunshineco.com> Helped-by: Junio C Hamano <gitster@pobox.com> Signed-off-by: René Scharfe <l.s.r@web.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
2023-04-06 22:19:11 +02:00
struct userdiff_driver *driver = userdiff_find_by_namelen(name, len);
if (driver && driver->word_regex_multi_byte) {
if (regexec_supports_multi_byte_chars())
driver->word_regex = driver->word_regex_multi_byte;
driver->word_regex_multi_byte = NULL;
}
return driver;
}
struct userdiff_driver *userdiff_find_by_path(struct index_state *istate,
const char *path)
{
static struct attr_check *check;
if (!check)
check = attr_check_initl("diff", NULL);
if (!path)
return NULL;
git_check_attr(istate, path, check);
if (ATTR_TRUE(check->items[0].value))
return &driver_true;
if (ATTR_FALSE(check->items[0].value))
return &driver_false;
if (ATTR_UNSET(check->items[0].value))
return NULL;
return userdiff_find_by_name(check->items[0].value);
}
struct userdiff_driver *userdiff_get_textconv(struct repository *r,
struct userdiff_driver *driver)
{
if (!driver->textconv)
return NULL;
if (driver->textconv_want_cache && !driver->textconv_cache &&
have_git_dir()) {
struct notes_cache *c = xmalloc(sizeof(*c));
struct strbuf name = STRBUF_INIT;
strbuf_addf(&name, "textconv/%s", driver->name);
notes_cache_init(r, c, name.buf, driver->textconv);
driver->textconv_cache = c;
strbuf_release(&name);
}
return driver;
}
static int for_each_userdiff_driver_list(each_userdiff_driver_fn fn,
enum userdiff_driver_type type, void *cb_data,
struct userdiff_driver *drv,
int drv_size)
{
int i;
int ret;
for (i = 0; i < drv_size; i++) {
struct userdiff_driver *item = drv + i;
if ((ret = fn(item, type, cb_data)))
return ret;
}
return 0;
}
int for_each_userdiff_driver(each_userdiff_driver_fn fn, void *cb_data)
{
int ret;
ret = for_each_userdiff_driver_list(fn, USERDIFF_DRIVER_TYPE_CUSTOM,
cb_data, drivers, ndrivers);
if (ret)
return ret;
ret = for_each_userdiff_driver_list(fn, USERDIFF_DRIVER_TYPE_BUILTIN,
cb_data, builtin_drivers,
ARRAY_SIZE(builtin_drivers));
if (ret)
return ret;
return 0;
}