From 332e9adf944e4ea232a855b1bf75ea4ddfd7e794 Mon Sep 17 00:00:00 2001

From: Ondrej Oprala <ooprala@redhat.com>

Date: Wed, 5 Aug 2015 09:15:09 +0200

Subject: [PATCH] expand,unexpand: add multibyte support

MIME-Version: 1.0

Content-Type: text/plain; charset=UTF-8

Content-Transfer-Encoding: 8bit

* NEWS: Mention the changes.

* bootstrap.conf: Add mbfile to the list of modules.

* configure.ac: Properly initialize mbfile.

* po/POTFILES.in: Add new source file.

* src/expand-core.c: Move functions common to both expand and

unexpand to this file.

* src/expand-core.h: Add function prototypes from expand-core.c.

* src/expand.c (expand): Iterate over multibyte characters properly.

* src/local.mk: Add expand-core.c to the lists of source codes for

expand and unexpand

* src/unexpand.c (unexpand): Iterate over multibyte characters

properly.

* tests/local.mk: Add new tests.

* tests/{expand,unexpand}/mb.sh: New tests.

Co-authored-by: Pádraig Brady <pbrady@redhat.com>

---

 NEWS                 |   3 +

 bootstrap.conf       |   1 +

 configure.ac         |   2 +

 po/POTFILES.in       |   1 +

 src/expand-core.c    | 150 +++++++++++++++++++++++++++++++++++++++

 src/expand-core.h    |  44 ++++++++++++

 src/expand.c         | 183 ++++++++++-------------------------------------

 src/local.mk         |   2 +

 src/unexpand.c       | 197 ++++++++++++---------------------------------------

 tests/expand/mb.sh   |  98 +++++++++++++++++++++++++

 tests/local.mk       |   2 +

 tests/unexpand/mb.sh |  97 +++++++++++++++++++++++++

 12 files changed, 482 insertions(+), 298 deletions(-)

 create mode 100644 src/expand-core.c

 create mode 100644 src/expand-core.h

 create mode 100755 tests/expand/mb.sh

 create mode 100755 tests/unexpand/mb.sh

diff --git a/bootstrap.conf b/bootstrap.conf

index ef1c078..ea8cebc 100644

--- a/bootstrap.conf

+++ b/bootstrap.conf

@@ -152,6 +152,7 @@ gnulib_modules="

   maintainer-makefile

   malloc-gnu

   manywarnings

+  mbfile

   mbrlen

   mbrtowc

   mbsalign

diff --git a/configure.ac b/configure.ac

index 8dc2192..b8b5114 100644

--- a/configure.ac

+++ b/configure.ac

@@ -422,6 +422,8 @@ gl_WINSIZE_IN_PTEM

 # I'm leaving it here for now.  This whole thing needs to be modernized...

 gl_WINSIZE_IN_PTEM

+gl_MBFILE

+

 gl_HEADER_TIOCGWINSZ_IN_TERMIOS_H

 if test $gl_cv_sys_tiocgwinsz_needs_termios_h = no && \

diff --git a/po/POTFILES.in b/po/POTFILES.in

index b3fe668..c594d20 100644

--- a/po/POTFILES.in

+++ b/po/POTFILES.in

@@ -57,6 +57,7 @@ src/dirname.c

 src/du.c

 src/echo.c

 src/env.c

+src/expand-core.c

 src/expand.c

 src/expr.c

 src/factor.c

diff --git a/src/expand-core.c b/src/expand-core.c

new file mode 100644

index 0000000..c8445db

--- /dev/null

+++ b/src/expand-core.c

@@ -0,0 +1,150 @@

+/* expand-core.c - elementary functions for the expand and unexpand utilities

+   Copyright (C) 1989-2015 Free Software Foundation, Inc.

+

+   This program is free software: you can redistribute it and/or modify

+   it under the terms of the GNU General Public License as published by

+   the Free Software Foundation, either version 3 of the License, or

+   (at your option) any later version.

+

+   This program is distributed in the hope that it will be useful,

+   but WITHOUT ANY WARRANTY; without even the implied warranty of

+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+   GNU General Public License for more details.

+

+   You should have received a copy of the GNU General Public License

+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */

+

+#include <config.h>

+

+#include <stdio.h>

+#include <sys/types.h>

+

+#include "system.h"

+#include "error.h"

+#include "fadvise.h"

+#include "quote.h"

+#include "xstrndup.h"

+

+#include "expand-core.h"

+

+/* Add the comma or blank separated list of tab stops STOPS

+   to the list of tab stops.  */

+

+extern void

+parse_tab_stops (char const *stops, void (*add_tab_stop)(uintmax_t))

+{

+  bool have_tabval = false;

+  uintmax_t tabval IF_LINT ( = 0);

+  char const *num_start IF_LINT ( = NULL);

+  bool ok = true;

+

+  for (; *stops; stops++)

+    {

+      if (*stops == ',' || isblank (to_uchar (*stops)))

+        {

+          if (have_tabval)

+            add_tab_stop (tabval);

+          have_tabval = false;

+        }

+      else if (ISDIGIT (*stops))

+        {

+          if (!have_tabval)

+            {

+              tabval = 0;

+              have_tabval = true;

+              num_start = stops;

+            }

+

+          /* Detect overflow.  */

+          if (!DECIMAL_DIGIT_ACCUMULATE (tabval, *stops - '0', uintmax_t))

+            {

+              size_t len = strspn (num_start, "0123456789");

+              char *bad_num = xstrndup (num_start, len);

+              error (0, 0, _("tab stop is too large %s"), quote (bad_num));

+              free (bad_num);

+              ok = false;

+              stops = num_start + len - 1;

+            }

+        }

+      else

+        {

+          error (0, 0, _("tab size contains invalid character(s): %s"),

+                 quote (stops));

+          ok = false;

+          break;

+        }

+    }

+

+  if (!ok)

+    exit (EXIT_FAILURE);

+

+  if (have_tabval)

+    add_tab_stop (tabval);

+}

+

+/* Check that the list of tab stops TABS, with ENTRIES entries,

+   contains only nonzero, ascending values.  */

+

+extern void

+validate_tab_stops (uintmax_t const *tabs, size_t entries)

+{

+  uintmax_t prev_tab = 0;

+  size_t i;

+

+  for (i = 0; i < entries; i++)

+    {

+      if (tabs[i] == 0)

+        error (EXIT_FAILURE, 0, _("tab size cannot be 0"));

+      if (tabs[i] <= prev_tab)

+        error (EXIT_FAILURE, 0, _("tab sizes must be ascending"));

+      prev_tab = tabs[i];

+    }

+}

+

+/* Close the old stream pointer FP if it is non-NULL,

+   and return a new one opened to read the next input file.

+   Open a filename of '-' as the standard input.

+   Return NULL if there are no more input files.  */

+

+extern FILE *

+next_file (FILE *fp)

+{

+  static char *prev_file;

+  char *file;

+

+  if (fp)

+    {

+      if (ferror (fp))

+        {

+          error (0, errno, "%s", prev_file);

+          exit_status = EXIT_FAILURE;

+        }

+      if (STREQ (prev_file, "-"))

+        clearerr (fp);		/* Also clear EOF.  */

+      else if (fclose (fp) != 0)

+        {

+          error (0, errno, "%s", prev_file);

+          exit_status = EXIT_FAILURE;

+        }

+    }

+

+  while ((file = *file_list++) != NULL)

+    {

+      if (STREQ (file, "-"))

+        {

+          have_read_stdin = true;

+          fp = stdin;

+        }

+      else

+        fp = fopen (file, "r");

+      if (fp)

+        {

+          prev_file = file;

+          fadvise (fp, FADVISE_SEQUENTIAL);

+          return fp;

+        }

+      error (0, errno, "%s", file);

+      exit_status = EXIT_FAILURE;

+    }

+  return NULL;

+}

diff --git a/src/expand-core.h b/src/expand-core.h

new file mode 100644

index 0000000..2419407

--- /dev/null

+++ b/src/expand-core.h

@@ -0,0 +1,41 @@

+/* expand-core.h - function prototypes for the expand and unexpand utilities

+   Copyright (C) 1989-2015 Free Software Foundation, Inc.

+

+   This program is free software: you can redistribute it and/or modify

+   it under the terms of the GNU General Public License as published by

+   the Free Software Foundation, either version 3 of the License, or

+   (at your option) any later version.

+

+   This program is distributed in the hope that it will be useful,

+   but WITHOUT ANY WARRANTY; without even the implied warranty of

+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+   GNU General Public License for more details.

+

+   You should have received a copy of the GNU General Public License

+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */

+

+#ifndef EXPAND_CORE_H_

+# define EXPAND_CORE_H_

+

+extern size_t first_free_tab;

+

+extern size_t n_tabs_allocated;

+

+extern uintmax_t *tab_list;

+

+extern int exit_status;

+

+extern char **file_list;

+

+extern bool have_read_stdin;

+

+void

+parse_tab_stops (char const *stops, void (*add_tab_stop)(uintmax_t));

+

+void

+validate_tab_stops (uintmax_t const *tabs, size_t entries);

+

+FILE *

+next_file (FILE *fp);

+

+#endif /* EXPAND_CORE_H_ */

diff --git a/src/expand.c b/src/expand.c

index 0a40a1a..ed97fd4 100644

--- a/src/expand.c

+++ b/src/expand.c

@@ -37,12 +37,16 @@

 #include <stdio.h>

 #include <getopt.h>

 #include <sys/types.h>

+

+#include <mbfile.h>

+

 #include "system.h"

 #include "error.h"

 #include "fadvise.h"

-#include "quote.h"

 #include "xstrndup.h"

+#include "expand-core.h"

+

 /* The official name of this program (e.g., no 'g' prefix).  */

 #define PROGRAM_NAME "expand"

@@ -58,17 +62,17 @@ static uintmax_t tab_size;

 /* Array of the explicit column numbers of the tab stops;

    after 'tab_list' is exhausted, each additional tab is replaced

    by a space.  The first column is column 0.  */

-static uintmax_t *tab_list;

+uintmax_t *tab_list;

 /* The number of allocated entries in 'tab_list'.  */

-static size_t n_tabs_allocated;

+size_t n_tabs_allocated;

 /* The index of the first invalid element of 'tab_list',

    where the next element can be added.  */

-static size_t first_free_tab;

+size_t first_free_tab;

 /* Null-terminated array of input filenames.  */

-static char **file_list;

+char **file_list;

 /* Default for 'file_list' if no files are given on the command line.  */

 static char *stdin_argv[] =

@@ -77,10 +81,10 @@ static char *stdin_argv[] =

 };

 /* True if we have ever read standard input.  */

-static bool have_read_stdin;

+bool have_read_stdin;

 /* The desired exit status.  */

-static int exit_status;

+int exit_status;

 static char const shortopts[] = "it:0::1::2::3::4::5::6::7::8::9::";

@@ -125,128 +129,6 @@

   if (first_free_tab == n_tabs_allocated)

     tab_list = X2NREALLOC (tab_list, &n_tabs_allocated);

   tab_list[first_free_tab++] = tabval;

-}

-

-/* Add the comma or blank separated list of tab stops STOPS

-   to the list of tab stops.  */

-

-static void

-parse_tab_stops (char const *stops)

-{

-  bool have_tabval = false;

-  uintmax_t tabval IF_LINT ( = 0);

-  char const *num_start IF_LINT ( = NULL);

-  bool ok = true;

-

-  for (; *stops; stops++)

-    {

-      if (*stops == ',' || isblank (to_uchar (*stops)))

-        {

-          if (have_tabval)

-            add_tab_stop (tabval);

-          have_tabval = false;

-        }

-      else if (ISDIGIT (*stops))

-        {

-          if (!have_tabval)

-            {

-              tabval = 0;

-              have_tabval = true;

-              num_start = stops;

-            }

-

-          /* Detect overflow.  */

-          if (!DECIMAL_DIGIT_ACCUMULATE (tabval, *stops - '0', uintmax_t))

-            {

-              size_t len = strspn (num_start, "0123456789");

-              char *bad_num = xstrndup (num_start, len);

-              error (0, 0, _("tab stop is too large %s"), quote (bad_num));

-              free (bad_num);

-              ok = false;

-              stops = num_start + len - 1;

-            }

-        }

-      else

-        {

-          error (0, 0, _("tab size contains invalid character(s): %s"),

-                 quote (stops));

-          ok = false;

-          break;

-        }

-    }

-

-  if (!ok)

-    exit (EXIT_FAILURE);

-

-  if (have_tabval)

-    add_tab_stop (tabval);

-}

-

-/* Check that the list of tab stops TABS, with ENTRIES entries,

-   contains only nonzero, ascending values.  */

-

-static void

-validate_tab_stops (uintmax_t const *tabs, size_t entries)

-{

-  uintmax_t prev_tab = 0;

-  size_t i;

-

-  for (i = 0; i < entries; i++)

-    {

-      if (tabs[i] == 0)

-        error (EXIT_FAILURE, 0, _("tab size cannot be 0"));

-      if (tabs[i] <= prev_tab)

-        error (EXIT_FAILURE, 0, _("tab sizes must be ascending"));

-      prev_tab = tabs[i];

-    }

-}

-

-/* Close the old stream pointer FP if it is non-NULL,

-   and return a new one opened to read the next input file.

-   Open a filename of '-' as the standard input.

-   Return NULL if there are no more input files.  */

-

-static FILE *

-next_file (FILE *fp)

-{

-  static char *prev_file;

-  char *file;

-

-  if (fp)

-    {

-      if (ferror (fp))

-        {

-          error (0, errno, "%s", quotef (prev_file));

-          exit_status = EXIT_FAILURE;

-        }

-      if (STREQ (prev_file, "-"))

-        clearerr (fp);		/* Also clear EOF.  */

-      else if (fclose (fp) != 0)

-        {

-          error (0, errno, "%s", quotef (prev_file));

-          exit_status = EXIT_FAILURE;

-        }

-    }

-

-  while ((file = *file_list++) != NULL)

-    {

-      if (STREQ (file, "-"))

-        {

-          have_read_stdin = true;

-          fp = stdin;

-        }

-      else

-        fp = fopen (file, "r");

-      if (fp)

-        {

-          prev_file = file;

-          fadvise (fp, FADVISE_SEQUENTIAL);

-          return fp;

-        }

-      error (0, errno, "%s", quotef (file));

-      exit_status = EXIT_FAILURE;

-    }

-  return NULL;

 }

 /* Change tabs to spaces, writing to stdout.

@@ -265,19 +146,19 @@ expand (void)

 {

   /* Input stream.  */

   FILE *fp = next_file (NULL);

+  mb_file_t mbf;

+  mbf_char_t c;

   if (!fp)

     return;

+  mbf_init (mbf, fp);

+

   while (true)

     {

-      /* Input character, or EOF.  */

-      int c;

-

       /* If true, perform translations.  */

       bool convert = true;

-

       /* The following variables have valid values only when CONVERT

          is true:  */

@@ -287,17 +168,23 @@ expand (void)

       /* Index in TAB_LIST of next tab stop to examine.  */

       size_t tab_index = 0;

-

       /* Convert a line of text.  */

       do

         {

-          while ((c = getc (fp)) < 0 && (fp = next_file (fp)))

-            continue;

+          do {

+            mbf_getc (c, mbf);

+            if (mb_iseof (c))

+              {

+                mbf_init (mbf, fp = next_file (fp));

+                continue;

+              }

+            }

+          while (false);

           if (convert)

             {

-              if (c == '\t')

+              if (mb_iseq (c, '\t'))

                 {

                   /* Column the next input tab stop is on.  */

                   uintmax_t next_tab_column;

@@ -328,32 +215,34 @@ expand (void)

                     if (putchar (' ') < 0)

                       error (EXIT_FAILURE, errno, _("write error"));

-                  c = ' ';

+                  mb_setascii (&c, ' ');

                 }

-              else if (c == '\b')

+              else if (mb_iseq (c, '\b'))

                 {

                   /* Go back one column, and force recalculation of the

                      next tab stop.  */

                   column -= !!column;

                   tab_index -= !!tab_index;

                 }

-              else

+              /* A leading control character could make us trip over.  */

+              else if (!mb_iscntrl (c))

                 {

-                  column++;

+                  column += mb_width (c);

                   if (!column)

                     error (EXIT_FAILURE, 0, _("input line is too long"));

                 }

-              convert &= convert_entire_line || !! isblank (c);

+              convert &= convert_entire_line || mb_isblank (c);

             }

-          if (c < 0)

+          if (mb_iseof (c))

             return;

-          if (putchar (c) < 0)

+          mb_putc (c, stdout);

+          if (ferror (stdout))

             error (EXIT_FAILURE, errno, _("write error"));

         }

-      while (c != '\n');

+      while (!mb_iseq (c, '\n'));

     }

 }

@@ -385,19 +274,19 @@ main (int argc, char **argv)

           break;

         case 't':

-          parse_tab_stops (optarg);

+          parse_tab_stops (optarg, add_tab_stop);

           break;

         case '0': case '1': case '2': case '3': case '4':

         case '5': case '6': case '7': case '8': case '9':

           if (optarg)

-            parse_tab_stops (optarg - 1);

+            parse_tab_stops (optarg - 1, add_tab_stop);

           else

             {

               char tab_stop[2];

               tab_stop[0] = c;

               tab_stop[1] = '\0';

-              parse_tab_stops (tab_stop);

+              parse_tab_stops (tab_stop, add_tab_stop);

             }

           break;

diff --git a/src/local.mk b/src/local.mk

index 536b7cc..bfede88 100644

--- a/src/local.mk

+++ b/src/local.mk

@@ -362,6 +362,8 @@ src_coreutils_SOURCES = src/coreutils.c

 src_cp_SOURCES = src/cp.c $(copy_sources) $(selinux_sources)

 src_dir_SOURCES = src/ls.c src/ls-dir.c

+src_expand_SOURCES = src/expand.c src/expand-core.c

+src_unexpand_SOURCES = src/unexpand.c src/expand-core.c

 src_vdir_SOURCES = src/ls.c src/ls-vdir.c

 src_id_SOURCES = src/id.c src/group-list.c

 src_groups_SOURCES = src/groups.c src/group-list.c

diff --git a/src/unexpand.c b/src/unexpand.c

index e0f7c22..48fbb32 100644

--- a/src/unexpand.c

+++ b/src/unexpand.c

@@ -38,12 +38,16 @@

 #include <stdio.h>

 #include <getopt.h>

 #include <sys/types.h>

+

+#include <mbfile.h>

+

 #include "system.h"

 #include "error.h"

 #include "fadvise.h"

-#include "quote.h"

 #include "xstrndup.h"

+#include "expand-core.h"

+

 /* The official name of this program (e.g., no 'g' prefix).  */

 #define PROGRAM_NAME "unexpand"

@@ -62,17 +66,17 @@ static size_t max_column_width;

 /* Array of the explicit column numbers of the tab stops;

    after 'tab_list' is exhausted, the rest of the line is printed

    unchanged.  The first column is column 0.  */

-static uintmax_t *tab_list;

+uintmax_t *tab_list;

 /* The number of allocated entries in 'tab_list'.  */

-static size_t n_tabs_allocated;

+size_t n_tabs_allocated;

 /* The index of the first invalid element of 'tab_list',

    where the next element can be added.  */

-static size_t first_free_tab;

+size_t first_free_tab;

 /* Null-terminated array of input filenames.  */

-static char **file_list;

+char **file_list;

 /* Default for 'file_list' if no files are given on the command line.  */

 static char *stdin_argv[] =

@@ -81,10 +85,10 @@ static char *stdin_argv[] =

 };

 /* True if we have ever read standard input.  */

-static bool have_read_stdin;

+bool have_read_stdin;

 /* The desired exit status.  */

-static int exit_status;

+int exit_status;

 /* For long options that have no equivalent short option, use a

    non-character as a pseudo short option, starting with CHAR_MAX + 1.  */

@@ -154,128 +156,6 @@ add_tab_stop (uintmax_t tabval)

     }

 }

-/* Add the comma or blank separated list of tab stops STOPS

-   to the list of tab stops.  */

-

-static void

-parse_tab_stops (char const *stops)

-{

-  bool have_tabval = false;

-  uintmax_t tabval IF_LINT ( = 0);

-  char const *num_start IF_LINT ( = NULL);

-  bool ok = true;

-

-  for (; *stops; stops++)

-    {

-      if (*stops == ',' || isblank (to_uchar (*stops)))

-        {

-          if (have_tabval)

-            add_tab_stop (tabval);

-          have_tabval = false;

-        }

-      else if (ISDIGIT (*stops))

-        {

-          if (!have_tabval)

-            {

-              tabval = 0;

-              have_tabval = true;

-              num_start = stops;

-            }

-

-          /* Detect overflow.  */

-          if (!DECIMAL_DIGIT_ACCUMULATE (tabval, *stops - '0', uintmax_t))

-            {

-              size_t len = strspn (num_start, "0123456789");

-              char *bad_num = xstrndup (num_start, len);

-              error (0, 0, _("tab stop is too large %s"), quote (bad_num));

-              free (bad_num);

-              ok = false;

-              stops = num_start + len - 1;

-            }

-        }

-      else

-        {

-          error (0, 0, _("tab size contains invalid character(s): %s"),

-                 quote (stops));

-          ok = false;

-          break;

-        }

-    }

-

-  if (!ok)

-    exit (EXIT_FAILURE);

-

-  if (have_tabval)

-    add_tab_stop (tabval);

-}

-

-/* Check that the list of tab stops TABS, with ENTRIES entries,

-   contains only nonzero, ascending values.  */

-

-static void

-validate_tab_stops (uintmax_t const *tabs, size_t entries)

-{

-  uintmax_t prev_tab = 0;

-  size_t i;

-

-  for (i = 0; i < entries; i++)

-    {

-      if (tabs[i] == 0)

-        error (EXIT_FAILURE, 0, _("tab size cannot be 0"));

-      if (tabs[i] <= prev_tab)

-        error (EXIT_FAILURE, 0, _("tab sizes must be ascending"));

-      prev_tab = tabs[i];

-    }

-}

-

-/* Close the old stream pointer FP if it is non-NULL,

-   and return a new one opened to read the next input file.

-   Open a filename of '-' as the standard input.

-   Return NULL if there are no more input files.  */

-

-static FILE *

-next_file (FILE *fp)

-{

-  static char *prev_file;

-  char *file;

-

-  if (fp)

-    {

-      if (ferror (fp))

-        {

-          error (0, errno, "%s", quotef (prev_file));

-          exit_status = EXIT_FAILURE;

-        }

-      if (STREQ (prev_file, "-"))

-        clearerr (fp);		/* Also clear EOF.  */

-      else if (fclose (fp) != 0)

-        {

-          error (0, errno, "%s", quotef (prev_file));

-          exit_status = EXIT_FAILURE;

-        }

-    }

-

-  while ((file = *file_list++) != NULL)

-    {

-      if (STREQ (file, "-"))

-        {

-          have_read_stdin = true;

-          fp = stdin;

-        }

-      else

-        fp = fopen (file, "r");

-      if (fp)

-        {

-          prev_file = file;

-          fadvise (fp, FADVISE_SEQUENTIAL);

-          return fp;

-        }

-      error (0, errno, "%s", quotef (file));

-      exit_status = EXIT_FAILURE;

-    }

-  return NULL;

-}

-

 /* Change blanks to tabs, writing to stdout.

    Read each file in 'file_list', in order.  */

@@ -284,11 +164,12 @@ unexpand (void)

 {

   /* Input stream.  */

   FILE *fp = next_file (NULL);

+  mb_file_t mbf;

   /* The array of pending blanks.  In non-POSIX locales, blanks can

      include characters other than spaces, so the blanks must be

      stored, not merely counted.  */

-  char *pending_blank;

+  mbf_char_t *pending_blank;

   if (!fp)

     return;

@@ -296,12 +177,14 @@ unexpand (void)

   /* The worst case is a non-blank character, then one blank, then a

      tab stop, then MAX_COLUMN_WIDTH - 1 blanks, then a non-blank; so

      allocate MAX_COLUMN_WIDTH bytes to store the blanks.  */

-  pending_blank = xmalloc (max_column_width);

+  pending_blank = xmalloc (max_column_width * sizeof (mbf_char_t));

+

+  mbf_init (mbf, fp);

   while (true)

     {

       /* Input character, or EOF.  */

-      int c;

+      mbf_char_t c;

       /* If true, perform translations.  */

       bool convert = true;

@@ -335,12 +218,19 @@ unexpand (void)

       do

         {

-          while ((c = getc (fp)) < 0 && (fp = next_file (fp)))

-            continue;

+          do {

+            mbf_getc (c, mbf);

+            if (mb_iseof (c))

+              {

+                mbf_init (mbf, fp = next_file (fp));

+                continue;

+              }

+            }

+          while (false);

           if (convert)

             {

-              bool blank = !! isblank (c);

+              bool blank = mb_isblank (c);

               if (blank)

                 {

@@ -372,16 +262,16 @@ unexpand (void)

                       if (next_tab_column < column)

                         error (EXIT_FAILURE, 0, _("input line is too long"));

-                      if (c == '\t')

+                      if (mb_iseq (c, '\t'))

                         {

                           column = next_tab_column;

                           if (pending)

-                            pending_blank[0] = '\t';

+                            mb_setascii (&pending_blank[0], '\t');

                         }

                       else

                         {

-                          column++;

+                          column += mb_width (c);

                           if (! (prev_blank && column == next_tab_column))

                             {

@@ -389,13 +279,14 @@ unexpand (void)

                                  will be replaced by tabs.  */

                               if (column == next_tab_column)

                                 one_blank_before_tab_stop = true;

-                              pending_blank[pending++] = c;

+                              mb_copy (&pending_blank[pending++], &c);

                               prev_blank = true;

                               continue;

                             }

                           /* Replace the pending blanks by a tab or two.  */

-                          pending_blank[0] = c = '\t';

+                          mb_setascii (&c, '\t');

+                          mb_setascii (&pending_blank[0], '\t');

                         }

                       /* Discard pending blanks, unless it was a single

@@ -403,7 +294,7 @@ unexpand (void)

                       pending = one_blank_before_tab_stop;