--- a/src/dfa.c +++ b/src/dfa.c @@ -1238,6 +1238,20 @@ parse_bracket_exp (void) return CSET + charclass_index (ccl); } +#define PUSH_LEX_STATE(s) \ + do \ + { \ + char const *lexptr_saved = lexptr; \ + size_t lexleft_saved = lexleft; \ + lexptr = (s); \ + lexleft = strlen (lexptr) + +#define POP_LEX_STATE() \ + lexptr = lexptr_saved; \ + lexleft = lexleft_saved; \ + } \ + while (0) + static token lex (void) { @@ -1485,20 +1499,6 @@ lex (void) return lasttok = CSET + charclass_index (ccl); } -#define PUSH_LEX_STATE(s) \ - do \ - { \ - char const *lexptr_saved = lexptr; \ - size_t lexleft_saved = lexleft; \ - lexptr = (s); \ - lexleft = strlen (lexptr) - -#define POP_LEX_STATE() \ - lexptr = lexptr_saved; \ - lexleft = lexleft_saved; \ - } \ - while (0) - /* FIXME: see if optimizing this, as is done with ANYCHAR and add_utf8_anychar, makes sense. */ @@ -1518,14 +1518,33 @@ lex (void) case 'W': if (!backslash || (syntax_bits & RE_NO_GNU_OPS)) goto normal_char; - zeroset (ccl); - for (c2 = 0; c2 < NOTCHAR; ++c2) - if (IS_WORD_CONSTITUENT (c2)) - setbit (c2, ccl); - if (c == 'W') - notset (ccl); + + if (!dfa->multibyte) + { + zeroset (ccl); + for (c2 = 0; c2 < NOTCHAR; ++c2) + if (IS_WORD_CONSTITUENT (c2)) + setbit (c2, ccl); + if (c == 'W') + notset (ccl); + laststart = false; + return lasttok = CSET + charclass_index (ccl); + } + + /* FIXME: see if optimizing this, as is done with ANYCHAR and + add_utf8_anychar, makes sense. */ + + /* \w and \W are documented to be equivalent to [_[:alnum:]] and + [^_[:alnum:]] respectively, so tell the lexer to process those + strings, each minus its "already processed" '['. */ + PUSH_LEX_STATE (c == 'w' ? "_[:alnum:]]" : "^_[:alnum:]]"); + + lasttok = parse_bracket_exp (); + + POP_LEX_STATE (); + laststart = false; - return lasttok = CSET + charclass_index (ccl); + return lasttok; case '[': if (backslash) --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -110,6 +110,7 @@ TESTS = \ warn-char-classes \ word-delim-multibyte \ word-multi-file \ + word-multibyte \ yesno EXTRA_DIST = \ --- a/tests/Makefile.in +++ b/tests/Makefile.in @@ -1409,6 +1409,7 @@ TESTS = \ warn-char-classes \ word-delim-multibyte \ word-multi-file \ + word-multibyte \ yesno EXTRA_DIST = \ @@ -2286,6 +2287,13 @@ word-multi-file.log: word-multi-file --log-file $$b.log --trs-file $$b.trs \ $(am__common_driver_flags) $(AM_LOG_DRIVER_FLAGS) $(LOG_DRIVER_FLAGS) -- $(LOG_COMPILE) \ "$$tst" $(AM_TESTS_FD_REDIRECT) +word-multibyte.log: word-multibyte + @p='word-multibyte'; \ + b='word-multibyte'; \ + $(am__check_pre) $(LOG_DRIVER) --test-name "$$f" \ + --log-file $$b.log --trs-file $$b.trs \ + $(am__common_driver_flags) $(AM_LOG_DRIVER_FLAGS) $(LOG_DRIVER_FLAGS) -- $(LOG_COMPILE) \ + "$$tst" $(AM_TESTS_FD_REDIRECT) yesno.log: yesno @p='yesno'; \ b='yesno'; \ --- a/dev/null +++ a/tests/word-multibyte @@ -0,0 +1,23 @@ +#!/bin/sh +# This would fail for grep-2.20 +. "${srcdir=.}/init.sh"; path_prepend_ ../src + +require_en_utf8_locale_ + +printf '\xc3\xa1\n' > in || framework_failure_ +LC_ALL=en_US.UTF-8 +export LC_ALL + +fail=0 + +for LOC in en_US.UTF-8 zh_CN $LOCALE_FR_UTF8; do + out=out1-$LOC + LC_ALL=$LOC grep '\w' in >$out || fail=1 + compare in $out || fail=1 + + out=out2-$LOC + LC_ALL=$LOC grep '\W' in >$out && fail=1 + compare /dev/null $out || fail=1 +done + +Exit $fail