From 29117b2d07af00f4d4b87cf778e4294588ab1a83 Mon Sep 17 00:00:00 2001

From: Kamil Dudka <kdudka@redhat.com>

Date: Thu, 1 Dec 2016 15:10:04 +0100

Subject: [PATCH] coreutils-i18n.patch

TODO: merge upstream

---

 lib/linebuffer.h            |   8 +

 src/fold.c                  | 308 +++++++++++++--

 src/join.c                  | 359 ++++++++++++++---

 src/pr.c                    | 443 +++++++++++++++++++--

 src/sort.c                  | 764 ++++++++++++++++++++++++++++++++++--

 src/uniq.c                  | 119 +++++-

 tests/i18n/sort.sh          |  29 ++

 tests/local.mk              |   2 +

 tests/misc/expand.pl        |  42 ++

 tests/misc/fold.pl          |  50 ++-

 tests/misc/join.pl          |  50 +++

 tests/misc/sort-mb-tests.sh |  45 +++

 tests/misc/sort-merge.pl    |  42 ++

 tests/misc/sort.pl          |  40 +-

 tests/misc/unexpand.pl      |  39 ++

 tests/misc/uniq.pl          |  55 +++

 tests/pr/pr-tests.pl        |  49 +++

 17 files changed, 2290 insertions(+), 154 deletions(-)

 create mode 100755 tests/i18n/sort.sh

 create mode 100755 tests/misc/sort-mb-tests.sh

diff --git a/lib/linebuffer.h b/lib/linebuffer.h

index 64181af..9b8fe5a 100644

--- a/lib/linebuffer.h

+++ b/lib/linebuffer.h

@@ -21,6 +21,11 @@

 # include <stdio.h>

+/* Get mbstate_t.  */

+# if HAVE_WCHAR_H

+#  include <wchar.h>

+# endif

+

 /* A 'struct linebuffer' holds a line of text. */

 struct linebuffer

@@ -28,6 +33,9 @@ struct linebuffer

   size_t size;                  /* Allocated. */

   size_t length;                /* Used. */

   char *buffer;

+# if HAVE_WCHAR_H

+  mbstate_t state;

+# endif

 };

 /* Initialize linebuffer LINEBUFFER for use. */

diff --git a/src/fold.c b/src/fold.c

index 8cd0d6b..d23edd5 100644

--- a/src/fold.c

+++ b/src/fold.c

@@ -22,12 +22,34 @@

 #include <getopt.h>

 #include <sys/types.h>

+/* Get mbstate_t, mbrtowc(), wcwidth().  */

+#if HAVE_WCHAR_H

+# include <wchar.h>

+#endif

+

+/* Get iswprint(), iswblank(), wcwidth().  */

+#if HAVE_WCTYPE_H

+# include <wctype.h>

+#endif

+

 #include "system.h"

 #include "die.h"

 #include "error.h"

 #include "fadvise.h"

 #include "xdectoint.h"

+/* MB_LEN_MAX is incorrectly defined to be 1 in at least one GCC

+      installation; work around this configuration error.  */

+#if !defined MB_LEN_MAX || MB_LEN_MAX < 2

+# undef MB_LEN_MAX

+# define MB_LEN_MAX 16

+#endif

+

+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t.  */

+#if HAVE_MBRTOWC && defined mbstate_t

+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)

+#endif

+

 #define TAB_WIDTH 8

 /* The official name of this program (e.g., no 'g' prefix).  */

@@ -35,20 +57,41 @@

 #define AUTHORS proper_name ("David MacKenzie")

+#define FATAL_ERROR(Message)                                            \

+  do                                                                    \

+    {                                                                   \

+      error (0, 0, (Message));                                          \

+      usage (2);                                                        \

+    }                                                                   \

+  while (0)

+

+enum operating_mode

+{

+  /* Fold texts by columns that are at the given positions. */

+  column_mode,

+

+  /* Fold texts by bytes that are at the given positions. */

+  byte_mode,

+

+  /* Fold texts by characters that are at the given positions. */

+  character_mode,

+};

+

+/* The argument shows current mode. (Default: column_mode) */

+static enum operating_mode operating_mode;

+

 /* If nonzero, try to break on whitespace. */

 static bool break_spaces;

-/* If nonzero, count bytes, not column positions. */

-static bool count_bytes;

-

 /* If nonzero, at least one of the files we read was standard input. */

 static bool have_read_stdin;

-static char const shortopts[] = "bsw:0::1::2::3::4::5::6::7::8::9::";

+static char const shortopts[] = "bcsw:0::1::2::3::4::5::6::7::8::9::";

 static struct option const longopts[] =

 {

   {"bytes", no_argument, NULL, 'b'},

+  {"characters", no_argument, NULL, 'c'},

   {"spaces", no_argument, NULL, 's'},

   {"width", required_argument, NULL, 'w'},

   {GETOPT_HELP_OPTION_DECL},

@@ -76,6 +119,7 @@ Wrap input lines in each FILE, writing to standard output.\n\

       fputs (_("\

   -b, --bytes         count bytes rather than columns\n\

+  -c, --characters    count characters rather than columns\n\

   -s, --spaces        break at spaces\n\

   -w, --width=WIDTH   use WIDTH columns instead of 80\n\

 "), stdout);

@@ -93,7 +137,7 @@ Wrap input lines in each FILE, writing to standard output.\n\

 static size_t

 adjust_column (size_t column, char c)

 {

-  if (!count_bytes)

+  if (operating_mode != byte_mode)

     {

       if (c == '\b')

         {

@@ -116,30 +160,14 @@ adjust_column (size_t column, char c)

    to stdout, with maximum line length WIDTH.

    Return true if successful.  */

-static bool

-fold_file (char const *filename, size_t width)

+static void

+fold_text (FILE *istream, size_t width, int *saved_errno)

 {

-  FILE *istream;

   int c;

   size_t column = 0;		/* Screen column where next char will go. */

   size_t offset_out = 0;	/* Index in 'line_out' for next char. */

   static char *line_out = NULL;

   static size_t allocated_out = 0;

-  int saved_errno;

-

-  if (STREQ (filename, "-"))

-    {

-      istream = stdin;

-      have_read_stdin = true;

-    }

-  else

-    istream = fopen (filename, "r");

-

-  if (istream == NULL)

-    {

-      error (0, errno, "%s", quotef (filename));

-      return false;

-    }

   fadvise (istream, FADVISE_SEQUENTIAL);

@@ -169,6 +197,15 @@ fold_file (char const *filename, size_t width)

               bool found_blank = false;

               size_t logical_end = offset_out;

+              /* If LINE_OUT has no wide character,

+                 put a new wide character in LINE_OUT

+                 if column is bigger than width. */

+              if (offset_out == 0)

+                {

+                  line_out[offset_out++] = c;

+                  continue;

+                }

+

               /* Look for the last blank. */

               while (logical_end)

                 {

@@ -215,11 +252,221 @@ fold_file (char const *filename, size_t width)

       line_out[offset_out++] = c;

     }

-  saved_errno = errno;

+  *saved_errno = errno;

   if (offset_out)

     fwrite (line_out, sizeof (char), (size_t) offset_out, stdout);

+}

+

+#if HAVE_MBRTOWC

+static void

+fold_multibyte_text (FILE *istream, size_t width, int *saved_errno)

+{

+  char buf[MB_LEN_MAX + BUFSIZ];  /* For spooling a read byte sequence. */

+  size_t buflen = 0;        /* The length of the byte sequence in buf. */

+  char *bufpos = buf;         /* Next read position of BUF. */

+  wint_t wc;                /* A gotten wide character. */

+  size_t mblength;        /* The byte size of a multibyte character which shows

+                           as same character as WC. */

+  mbstate_t state, state_bak;        /* State of the stream. */

+  int convfail = 0;                /* 1, when conversion is failed. Otherwise 0. */

+

+  static char *line_out = NULL;

+  size_t offset_out = 0;        /* Index in `line_out' for next char. */

+  static size_t allocated_out = 0;

+

+  int increment;

+  size_t column = 0;

+

+  size_t last_blank_pos;

+  size_t last_blank_column;

+  int is_blank_seen;

+  int last_blank_increment = 0;

+  int is_bs_following_last_blank;

+  size_t bs_following_last_blank_num;

+  int is_cr_after_last_blank;

+

+#define CLEAR_FLAGS                                \

+   do                                                \

+     {                                                \

+        last_blank_pos = 0;                        \

+        last_blank_column = 0;                        \

+        is_blank_seen = 0;                        \

+        is_bs_following_last_blank = 0;                \

+        bs_following_last_blank_num = 0;        \

+        is_cr_after_last_blank = 0;                \

+     }                                                \

+   while (0)

+

+#define START_NEW_LINE                        \

+   do                                        \

+     {                                        \

+      putchar ('\n');                        \

+      column = 0;                        \

+      offset_out = 0;                        \

+      CLEAR_FLAGS;                        \

+    }                                        \

+   while (0)

+

+  CLEAR_FLAGS;

+  memset (&state, '\0', sizeof(mbstate_t));

+

+  for (;; bufpos += mblength, buflen -= mblength)

+    {

+      if (buflen < MB_LEN_MAX && !feof (istream) && !ferror (istream))

+        {

+          memmove (buf, bufpos, buflen);

+          buflen += fread (buf + buflen, sizeof(char), BUFSIZ, istream);

+          bufpos = buf;

+        }

+

+      if (buflen < 1)

+        break;

+

+      /* Get a wide character. */

+      state_bak = state;

+      mblength = mbrtowc ((wchar_t *)&wc, bufpos, buflen, &state);

+

+      switch (mblength)

+        {

+        case (size_t)-1:

+        case (size_t)-2:

+          convfail++;

+          state = state_bak;

+          /* Fall through. */

+

+        case 0:

+          mblength = 1;

+          break;

+        }

+

+rescan:

+      if (operating_mode == byte_mode)                        /* byte mode */

+        increment = mblength;

+      else if (operating_mode == character_mode)        /* character mode */

+        increment = 1;

+      else                                                /* column mode */

+        {

+          if (convfail)

+            increment = 1;

+          else

+            {

+              switch (wc)

+                {

+                case L'\n':

+                  fwrite (line_out, sizeof(char), offset_out, stdout);

+                  START_NEW_LINE;

+                  continue;

+

+                case L'\b':

+                  increment = (column > 0) ? -1 : 0;

+                  break;

+

+                case L'\r':

+                  increment = -1 * column;

+                  break;

+

+                case L'\t':

+                  increment = 8 - column % 8;

+                  break;

+

+                default:

+                  increment = wcwidth (wc);

+                  increment = (increment < 0) ? 0 : increment;

+                }

+            }

+        }

+

+      if (column + increment > width && break_spaces && last_blank_pos)

+        {

+          fwrite (line_out, sizeof(char), last_blank_pos, stdout);

+          putchar ('\n');

+

+          offset_out = offset_out - last_blank_pos;

+          column = column - last_blank_column + ((is_cr_after_last_blank)

+              ? last_blank_increment : bs_following_last_blank_num);

+          memmove (line_out, line_out + last_blank_pos, offset_out);

+          CLEAR_FLAGS;

+          goto rescan;

+        }

+

+      if (column + increment > width && column != 0)

+        {

+          fwrite (line_out, sizeof(char), offset_out, stdout);

+          START_NEW_LINE;

+          goto rescan;

+        }

+

+      if (allocated_out < offset_out + mblength)

+        {

+          line_out = X2REALLOC (line_out, &allocated_out);

+        }

+

+      memcpy (line_out + offset_out, bufpos, mblength);

+      offset_out += mblength;

+      column += increment;

+

+      if (is_blank_seen && !convfail && wc == L'\r')

+        is_cr_after_last_blank = 1;

+

+      if (is_bs_following_last_blank && !convfail && wc == L'\b')

+        ++bs_following_last_blank_num;

+      else

+        is_bs_following_last_blank = 0;

+

+      if (break_spaces && !convfail && iswblank (wc))

+        {

+          last_blank_pos = offset_out;

+          last_blank_column = column;

+          is_blank_seen = 1;

+          last_blank_increment = increment;

+          is_bs_following_last_blank = 1;

+          bs_following_last_blank_num = 0;

+          is_cr_after_last_blank = 0;

+        }

+    }

+

+  *saved_errno = errno;

+

+  if (offset_out)

+    fwrite (line_out, sizeof (char), (size_t) offset_out, stdout);

+

+}

+#endif

+

+/* Fold file FILENAME, or standard input if FILENAME is "-",

+   to stdout, with maximum line length WIDTH.

+   Return 0 if successful, 1 if an error occurs. */

+

+static bool

+fold_file (char const *filename, size_t width)

+{

+  FILE *istream;

+  int saved_errno;

+

+  if (STREQ (filename, "-"))

+    {

+      istream = stdin;

+      have_read_stdin = 1;

+    }

+  else

+    istream = fopen (filename, "r");

+

+  if (istream == NULL)

+    {

+      error (0, errno, "%s", filename);

+      return 1;

+    }

+

+  /* Define how ISTREAM is being folded. */

+#if HAVE_MBRTOWC

+  if (MB_CUR_MAX > 1)

+    fold_multibyte_text (istream, width, &saved_errno);

+  else

+#endif

+    fold_text (istream, width, &saved_errno);

+

   if (ferror (istream))

     {

       error (0, saved_errno, "%s", quotef (filename));

@@ -252,7 +499,8 @@ main (int argc, char **argv)

   atexit (close_stdout);

-  break_spaces = count_bytes = have_read_stdin = false;

+  operating_mode = column_mode;

+  break_spaces = have_read_stdin = false;

   while ((optc = getopt_long (argc, argv, shortopts, longopts, NULL)) != -1)

     {

@@ -261,7 +509,15 @@ main (int argc, char **argv)

       switch (optc)

         {

         case 'b':		/* Count bytes rather than columns. */

-          count_bytes = true;

+          if (operating_mode != column_mode)

+            FATAL_ERROR (_("only one way of folding may be specified"));

+          operating_mode = byte_mode;

+          break;

+

+        case 'c':

+          if (operating_mode != column_mode)

+            FATAL_ERROR (_("only one way of folding may be specified"));

+          operating_mode = character_mode;

           break;

         case 's':		/* Break at word boundaries. */

diff --git a/src/join.c b/src/join.c

index 98b461c..9990f38 100644

--- a/src/join.c

+++ b/src/join.c

@@ -22,19 +22,33 @@

 #include <sys/types.h>

 #include <getopt.h>

+/* Get mbstate_t, mbrtowc(), mbrtowc(), wcwidth().  */

+#if HAVE_WCHAR_H

+# include <wchar.h>

+#endif

+

+/* Get iswblank(), towupper.  */

+#if HAVE_WCTYPE_H

+# include <wctype.h>

+#endif

+

 #include "system.h"

 #include "die.h"

 #include "error.h"

 #include "fadvise.h"

 #include "hard-locale.h"

 #include "linebuffer.h"

-#include "memcasecmp.h"

 #include "quote.h"

 #include "stdio--.h"

 #include "xmemcoll.h"

 #include "xstrtol.h"

 #include "argmatch.h"

+/* Some systems, like BeOS, have multibyte encodings but lack mbstate_t.  */

+#if HAVE_MBRTOWC && defined mbstate_t

+# define mbrtowc(pwc, s, n, ps) (mbrtowc) (pwc, s, n, 0)

+#endif

+

 /* The official name of this program (e.g., no 'g' prefix).  */

 #define PROGRAM_NAME "join"

@@ -136,10 +150,12 @@ static struct outlist outlist_head;

 /* Last element in 'outlist', where a new element can be added.  */

 static struct outlist *outlist_end = &outlist_head;

-/* Tab character separating fields.  If negative, fields are separated

-   by any nonempty string of blanks, otherwise by exactly one

-   tab character whose value (when cast to unsigned char) equals TAB.  */

-static int tab = -1;

+/* Tab character separating fields.  If NULL, fields are separated

+   by any nonempty string of blanks.  */

+static char *tab = NULL;

+

+/* The number of bytes used for tab. */

+static size_t tablen = 0;

 /* If nonzero, check that the input is correctly ordered. */

 static enum

@@ -276,13 +292,14 @@ xfields (struct line *line)

   if (ptr == lim)

     return;

-  if (0 <= tab && tab != '\n')

+  if (tab != NULL)

     {

+      unsigned char t = tab[0];

       char *sep;

-      for (; (sep = memchr (ptr, tab, lim - ptr)) != NULL; ptr = sep + 1)

+      for (; (sep = memchr (ptr, t, lim - ptr)) != NULL; ptr = sep + 1)

         extract_field (line, ptr, sep - ptr);

     }

-  else if (tab < 0)

+   else

     {

       /* Skip leading blanks before the first field.  */

       while (field_sep (*ptr))

@@ -306,6 +323,147 @@ xfields (struct line *line)

   extract_field (line, ptr, lim - ptr);

 }

+#if HAVE_MBRTOWC

+static void

+xfields_multibyte (struct line *line)

+{

+  char *ptr = line->buf.buffer;

+  char const *lim = ptr + line->buf.length - 1;

+  wchar_t wc = 0;

+  size_t mblength = 1;

+  mbstate_t state, state_bak;

+

+  memset (&state, 0, sizeof (mbstate_t));

+

+  if (ptr >= lim)

+    return;

+

+  if (tab != NULL)

+    {

+      char *sep = ptr;

+      for (; ptr < lim; ptr = sep + mblength)

+	{

+	  sep = ptr;

+	  while (sep < lim)

+	    {

+	      state_bak = state;

+	      mblength = mbrtowc (&wc, sep, lim - sep + 1, &state);

+

+	      if (mblength == (size_t)-1 || mblength == (size_t)-2)

+		{

+		  mblength = 1;

+		  state = state_bak;

+		}

+	      mblength = (mblength < 1) ? 1 : mblength;

+

+	      if (mblength == tablen && !memcmp (sep, tab, mblength))

+		break;

+	      else

+		{

+		  sep += mblength;

+		  continue;

+		}

+	    }

+

+	  if (sep >= lim)

+	    break;

+

+	  extract_field (line, ptr, sep - ptr);

+	}

+    }

+  else

+    {

+      /* Skip leading blanks before the first field.  */

+      while(ptr < lim)

+      {

+        state_bak = state;

+        mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state);

+

+        if (mblength == (size_t)-1 || mblength == (size_t)-2)

+          {

+            mblength = 1;

+            state = state_bak;

+            break;

+          }

+        mblength = (mblength < 1) ? 1 : mblength;

+

+        if (!iswblank(wc) && wc != '\n')

+          break;

+        ptr += mblength;

+      }

+

+      do

+	{

+	  char *sep;

+	  state_bak = state;

+	  mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state);

+	  if (mblength == (size_t)-1 || mblength == (size_t)-2)

+	    {

+	      mblength = 1;

+	      state = state_bak;

+	      break;

+	    }

+	  mblength = (mblength < 1) ? 1 : mblength;

+

+	  sep = ptr + mblength;

+	  while (sep < lim)

+	    {

+	      state_bak = state;

+	      mblength = mbrtowc (&wc, sep, lim - sep + 1, &state);

+	      if (mblength == (size_t)-1 || mblength == (size_t)-2)

+		{

+		  mblength = 1;

+		  state = state_bak;

+		  break;

+		}

+	      mblength = (mblength < 1) ? 1 : mblength;

+

+	      if (iswblank (wc) || wc == '\n')

+		break;

+

+	      sep += mblength;

+	    }

+

+	  extract_field (line, ptr, sep - ptr);

+	  if (sep >= lim)

+	    return;

+

+	  state_bak = state;

+	  mblength = mbrtowc (&wc, sep, lim - sep + 1, &state);

+	  if (mblength == (size_t)-1 || mblength == (size_t)-2)

+	    {

+	      mblength = 1;

+	      state = state_bak;

+	      break;

+	    }

+	  mblength = (mblength < 1) ? 1 : mblength;

+

+	  ptr = sep + mblength;

+	  while (ptr < lim)

+	    {

+	      state_bak = state;

+	      mblength = mbrtowc (&wc, ptr, lim - ptr + 1, &state);

+	      if (mblength == (size_t)-1 || mblength == (size_t)-2)

+		{

+		  mblength = 1;

+		  state = state_bak;

+		  break;

+		}

+	      mblength = (mblength < 1) ? 1 : mblength;

+

+	      if (!iswblank (wc) && wc != '\n')

+		break;

+

+	      ptr += mblength;

+	    }

+	}

+      while (ptr < lim);

+    }

+

+  extract_field (line, ptr, lim - ptr);

+}

+#endif

+

 static void

 freeline (struct line *line)

 {

@@ -327,56 +485,133 @@ keycmp (struct line const *line1, struct line const *line2,

         size_t jf_1, size_t jf_2)

 {

   /* Start of field to compare in each file.  */

-  char *beg1;

-  char *beg2;

-

-  size_t len1;

-  size_t len2;		/* Length of fields to compare.  */

+  char *beg[2];

+  char *copy[2];

+  size_t len[2]; 	/* Length of fields to compare.  */

   int diff;

+  int i, j;

+  int mallocd = 0;

   if (jf_1 < line1->nfields)

     {

-      beg1 = line1->fields[jf_1].beg;

-      len1 = line1->fields[jf_1].len;

+      beg[0] = line1->fields[jf_1].beg;

+      len[0] = line1->fields[jf_1].len;

     }

   else

     {

-      beg1 = NULL;

-      len1 = 0;

+      beg[0] = NULL;

+      len[0] = 0;

     }

   if (jf_2 < line2->nfields)

     {

-      beg2 = line2->fields[jf_2].beg;

-      len2 = line2->fields[jf_2].len;

+      beg[1] = line2->fields[jf_2].beg;

+      len[1] = line2->fields[jf_2].len;

     }

   else

     {

-      beg2 = NULL;

-      len2 = 0;