From af2a4ed22594badd2719c0123441d69b17bd8328 Mon Sep 17 00:00:00 2001

From: Federico Simoncelli <fsimonce@redhat.com>

Date: Fri, 26 Sep 2014 17:12:32 +0000

Subject: dd: new status=progress level to print stats periodically

* src/dd.c: Report the transfer progress every second when the

new status=progress level is used.  Adjust the handling and

description of the status= option so that they're treated as

mutually exclusive levels, rather than flags with implicit precedence.

* doc/coreutils.texi (dd invocation): Document the new progress

status level.  Reference the new level in the description of SIGUSR1.

* tests/dd/stats.sh: Add new test for status=progress.

* tests/dd/misc.sh: Change so status=none only takes precedence

if it's the last level specified.

---

diff --git a/doc/coreutils.texi b/doc/coreutils.texi

index 7d32af5..03bb710 100644

--- a/doc/coreutils.texi

+++ b/doc/coreutils.texi

@@ -8631,24 +8631,32 @@ will ensure that @samp{count=} corresponds to complete input blocks

 rather than the traditional POSIX specified behavior of counting

 input read operations.

-@item status=@var{which}

+@item status=@var{level}

 @opindex status

 Transfer information is normally output to stderr upon

 receipt of the @samp{INFO} signal or when @command{dd} exits.

-Specifying @var{which} will identify which information to suppress.

+Specifying @var{level} will adjust the amount of information printed,

+with the last @var{level} specified taking precedence.

 @table @samp

-@item noxfer

-@opindex noxfer @r{dd status=}

-Do not print the transfer rate and volume statistics

-that normally make up the last status line.

-

 @item none

 @opindex none @r{dd status=}

 Do not print any informational or warning messages to stderr.

 Error messages are output as normal.

+@item noxfer

+@opindex noxfer @r{dd status=}

+Do not print the final transfer rate and volume statistics

+that normally make up the last status line.

+

+@item progress

+@opindex progress @r{dd status=}

+Print the transfer rate and volume statistics on stderr,

+when processing each input block.  Statistics are output

+on a single line at most once every second, but updates

+can be delayed when waiting on I/O.

+

 @end table

 @item conv=@var{conversion}[,@var{conversion}]@dots{}

@@ -9033,6 +9041,9 @@ The above script will output in the following format

 5120000000 bytes (5.1 GB) copied, 18.913 seconds, 271 MB/s

 @end example

+Note also the @samp{status=progress} option which periodically updates

+the last line of the transfer statistics above.

+

 @vindex POSIXLY_CORRECT

 On systems lacking the @samp{INFO} signal @command{dd} responds to the

 @samp{USR1} signal instead, unless the @env{POSIXLY_CORRECT}

diff --git a/src/dd.c b/src/dd.c

index d22ec59..4018190 100644

--- a/src/dd.c

+++ b/src/dd.c

@@ -34,6 +34,7 @@

 #include "long-options.h"

 #include "quote.h"

 #include "quotearg.h"

+#include "verror.h"

 #include "xstrtol.h"

 #include "xtime.h"

@@ -132,11 +133,13 @@ enum

     C_SPARSE = 0200000

   };

-/* Status bit masks.  */

+/* Status levels.  */

 enum

   {

-    STATUS_NOXFER = 01,

-    STATUS_NONE = 02

+    STATUS_NONE = 1,

+    STATUS_NOXFER = 2,

+    STATUS_DEFAULT = 3,

+    STATUS_PROGRESS = 4

   };

 /* The name of the input file, or NULL for the standard input. */

@@ -188,7 +191,7 @@ static int input_flags = 0;

 static int output_flags = 0;

 /* Status flags for what is printed to stderr.  */

-static int status_flags = 0;

+static int status_level = STATUS_DEFAULT;

 /* If nonzero, filter characters through the translation table.  */

 static bool translation_needed = false;

@@ -211,6 +214,12 @@ static uintmax_t w_bytes = 0;

 /* Time that dd started.  */

 static xtime_t start_time;

+/* Previous time for periodic progress.  */

+static xtime_t previous_time;

+

+/* Whether a '\n' is pending after writing progress.  */

+static bool newline_pending;

+

 /* True if input is seekable.  */

 static bool input_seekable;

@@ -373,8 +382,9 @@ static struct symbol_value const flags[] =

 /* Status, for status="...".  */

 static struct symbol_value const statuses[] =

 {

-  {"noxfer",	STATUS_NOXFER},

   {"none",	STATUS_NONE},

+  {"noxfer",	STATUS_NOXFER},

+  {"progress",	STATUS_PROGRESS},

   {"",		0}

 };

@@ -517,6 +527,25 @@ maybe_close_stdout (void)

     _exit (EXIT_FAILURE);

 }

+/* Like error() but handle any pending newline.  */

+

+static void _GL_ATTRIBUTE_FORMAT ((__printf__, 3, 4))

+nl_error (int status, int errnum, const char *fmt, ...)

+{

+  if (newline_pending)

+    {

+      fputc ('\n', stderr);

+      newline_pending = false;

+    }

+

+  va_list ap;

+  va_start (ap, fmt);

+  verror (status, errnum, fmt, ap);

+  va_end (ap);

+}

+

+#define error nl_error

+

 void

 usage (int status)

 {

@@ -546,8 +575,10 @@ Copy a file, converting and formatting according to the operands.\n\

   oflag=FLAGS     write as per the comma separated symbol list\n\

   seek=N          skip N obs-sized blocks at start of output\n\

   skip=N          skip N ibs-sized blocks at start of input\n\

-  status=WHICH    WHICH info to suppress outputting to stderr;\n\

-                  'noxfer' suppresses transfer stats, 'none' suppresses all\n\

+  status=LEVEL    The LEVEL of information to print to stderr;\n\

+                  'none' suppresses everything but error messages,\n\

+                  'noxfer' suppresses the final transfer statistics,\n\

+                  'progress' shows periodic transfer statistics\n\

 "), stdout);

       fputs (_("\

 \n\

@@ -724,8 +755,7 @@ multiple_bits_set (int i)

 /* Print transfer statistics.  */

 static void

-print_stats (void)

-{

+print_xfer_stats (xtime_t progress_time) {

   char hbuf[LONGEST_HUMAN_READABLE + 1];

   int human_opts =

     (human_autoscale | human_round_to_nearest

@@ -733,23 +763,8 @@ print_stats (void)

   double delta_s;

   char const *bytes_per_second;

-  if (status_flags & STATUS_NONE)

-    return;

-

-  fprintf (stderr,

-           _("%"PRIuMAX"+%"PRIuMAX" records in\n"

-             "%"PRIuMAX"+%"PRIuMAX" records out\n"),

-           r_full, r_partial, w_full, w_partial);

-

-  if (r_truncate != 0)

-    fprintf (stderr,

-             ngettext ("%"PRIuMAX" truncated record\n",

-                       "%"PRIuMAX" truncated records\n",

-                       select_plural (r_truncate)),

-             r_truncate);

-

-  if (status_flags & STATUS_NOXFER)

-    return;

+  if (progress_time)

+    fputc ('\r', stderr);

   /* Use integer arithmetic to compute the transfer rate,

      since that makes it easy to use SI abbreviations.  */

@@ -761,7 +776,8 @@ print_stats (void)

            w_bytes,

            human_readable (w_bytes, hbuf, human_opts, 1, 1));

-  xtime_t now = gethrxtime ();

+  xtime_t now = progress_time ? progress_time : gethrxtime ();

+

   if (start_time < now)

     {

       double XTIME_PRECISIONe0 = XTIME_PRECISION;

@@ -787,7 +803,42 @@ print_stats (void)

      but that was incorrect for languages like Polish.  To fix this

      bug we now use SI symbols even though they're a bit more

      confusing in English.  */

-  fprintf (stderr, _(", %g s, %s/s\n"), delta_s, bytes_per_second);

+  char const *time_fmt = _(", %g s, %s/s\n");;

+  if (progress_time)

+    time_fmt = _(", %.6f s, %s/s");  /* OK with '\r' as increasing width.  */

+  fprintf (stderr, time_fmt, delta_s, bytes_per_second);

+

+  newline_pending = !!progress_time;

+}

+

+static void

+print_stats (void)

+{

+  if (status_level == STATUS_NONE)

+    return;

+

+  if (newline_pending)

+    {

+      fputc ('\n', stderr);

+      newline_pending = false;

+    }

+

+  fprintf (stderr,

+           _("%"PRIuMAX"+%"PRIuMAX" records in\n"

+             "%"PRIuMAX"+%"PRIuMAX" records out\n"),

+           r_full, r_partial, w_full, w_partial);

+

+  if (r_truncate != 0)

+    fprintf (stderr,

+             ngettext ("%"PRIuMAX" truncated record\n",

+                       "%"PRIuMAX" truncated records\n",

+                       select_plural (r_truncate)),

+             r_truncate);

+

+  if (status_level == STATUS_NOXFER)

+    return;

+

+  print_xfer_stats (0);

 }

 /* An ordinary signal was received; arrange for the program to exit.  */

@@ -1035,7 +1086,7 @@ iread (int fd, char *buf, size_t size)

       if (0 < prev_nread && prev_nread < size)

         {

           uintmax_t prev = prev_nread;

-          if (!(status_flags & STATUS_NONE))

+          if (status_level != STATUS_NONE)

             error (0, 0, ngettext (("warning: partial read (%"PRIuMAX" byte); "

                                     "suggest iflag=fullblock"),

                                    ("warning: partial read (%"PRIuMAX" bytes); "

@@ -1086,7 +1137,7 @@ iwrite (int fd, char const *buf, size_t size)

     {

       int old_flags = fcntl (STDOUT_FILENO, F_GETFL);

       if (fcntl (STDOUT_FILENO, F_SETFL, old_flags & ~O_DIRECT) != 0

-          && !(status_flags & STATUS_NONE))

+          && status_level != STATUS_NONE)

         error (0, errno, _("failed to turn off O_DIRECT: %s"),

                quote (output_file));

@@ -1219,7 +1270,7 @@ operand_matches (char const *str, char const *pattern, char delim)

 static int

 parse_symbols (char const *str, struct symbol_value const *table,

-               char const *error_msgid)

+               bool exclusive, char const *error_msgid)

 {

   int value = 0;

@@ -1241,7 +1292,10 @@ parse_symbols (char const *str, struct symbol_value const *table,

             }

         }

-      value |= entry->value;

+      if (exclusive)

+        value = entry->value;

+      else

+        value |= entry->value;

       if (!strcomma)

         break;

       str = strcomma + 1;

@@ -1316,17 +1370,17 @@ scanargs (int argc, char *const *argv)

       else if (operand_is (name, "of"))

         output_file = val;

       else if (operand_is (name, "conv"))

-        conversions_mask |= parse_symbols (val, conversions,

+        conversions_mask |= parse_symbols (val, conversions, false,

                                            N_("invalid conversion"));

       else if (operand_is (name, "iflag"))

-        input_flags |= parse_symbols (val, flags,

+        input_flags |= parse_symbols (val, flags, false,

                                       N_("invalid input flag"));

       else if (operand_is (name, "oflag"))

-        output_flags |= parse_symbols (val, flags,

+        output_flags |= parse_symbols (val, flags, false,

                                        N_("invalid output flag"));

       else if (operand_is (name, "status"))

-        status_flags |= parse_symbols (val, statuses,

-                                       N_("invalid status flag"));

+        status_level = parse_symbols (val, statuses, true,

+                                      N_("invalid status level"));

       else

         {

           bool invalid = false;

@@ -1613,7 +1667,7 @@ skip_via_lseek (char const *filename, int fdesc, off_t offset, int whence)

       && ioctl (fdesc, MTIOCGET, &s2) == 0

       && MT_SAME_POSITION (s1, s2))

     {

-      if (!(status_flags & STATUS_NONE))

+      if (status_level != STATUS_NONE)

         error (0, 0, _("warning: working around lseek kernel bug for file "

                        "(%s)\n  of mt_type=0x%0lx -- "

                        "see <sys/mtio.h> for the list of types"),

@@ -1787,7 +1841,7 @@ advance_input_after_read_error (size_t nbytes)

           if (offset == input_offset)

             return true;

           diff = input_offset - offset;

-          if (! (0 <= diff && diff <= nbytes) && !(status_flags & STATUS_NONE))

+          if (! (0 <= diff && diff <= nbytes) && status_level != STATUS_NONE)

             error (0, 0, _("warning: invalid file offset after failed read"));

           if (0 <= skip_via_lseek (input_file, STDIN_FILENO, diff, SEEK_CUR))

             return true;

@@ -1986,7 +2040,7 @@ dd_copy (void)

              2. pipe has not enough data

              3. partial reads  */

       if ((us_blocks || (!input_offset_overflow && us_bytes))

-          && !(status_flags & STATUS_NONE))

+          && status_level != STATUS_NONE)

         {

           error (0, 0,

                  _("%s: cannot skip to specified offset"), quote (input_file));

@@ -2029,6 +2083,19 @@ dd_copy (void)

   while (1)

     {

+      if (status_level == STATUS_PROGRESS)

+        {

+          xtime_t progress_time = gethrxtime ();

+          uintmax_t delta_xtime = progress_time;

+          delta_xtime -= previous_time;

+          double XTIME_PRECISIONe0 = XTIME_PRECISION;

+          if (delta_xtime / XTIME_PRECISIONe0 > 1)

+            {

+              print_xfer_stats (progress_time);

+              previous_time = progress_time;

+            }

+        }

+

       if (r_partial + r_full >= max_records + !!max_bytes)

         break;

@@ -2053,7 +2120,7 @@ dd_copy (void)

       if (nread < 0)

         {

-          if (!(conversions_mask & C_NOERROR) || !(status_flags & STATUS_NONE))

+          if (!(conversions_mask & C_NOERROR) || status_level != STATUS_NONE)

             error (0, errno, _("error reading %s"), quote (input_file));

           if (conversions_mask & C_NOERROR)

@@ -2345,7 +2412,7 @@ main (int argc, char **argv)

         }

     }

-  start_time = gethrxtime ();

+  start_time = previous_time = gethrxtime ();

   exit_status = dd_copy ();

diff --git a/tests/dd/misc.sh b/tests/dd/misc.sh

index f877fdd..34dfba7 100755

--- a/tests/dd/misc.sh

+++ b/tests/dd/misc.sh

@@ -35,9 +35,12 @@ dd status=none if=$tmp_in of=/dev/null 2> err || fail=1

 test -s err && { cat err; fail=1; }

 dd status=none if=$tmp_in skip=2 of=/dev/null 2> err || fail=1

 test -s err && { cat err; fail=1; }

-# check status=none is cumulative with status=noxfer

-dd status=none status=noxfer if=$tmp_in of=/dev/null 2> err || fail=1

+# check later status=none overrides earlier status=noxfer

+dd status=noxfer status=none if=$tmp_in of=/dev/null 2> err || fail=1

 test -s err && { cat err; fail=1; }

+# check later status=noxfer overrides earlier status=none

+dd status=none status=noxfer if=$tmp_in of=/dev/null 2> err || fail=1

+compare /dev/null err && fail=1

 dd if=$tmp_in of=$tmp_out 2> /dev/null || fail=1

 compare $tmp_in $tmp_out || fail=1

diff --git a/tests/dd/stats.sh b/tests/dd/stats.sh

new file mode 100755

index 0000000..24b8c49`

--- a/dev/null

+++ b/tests/dd/stats.sh

@@ -0,0 +1,65 @@

+#!/bin/sh

+# Check stats output for SIG{INFO,USR1} and status=progress

+

+# Copyright (C) 2014 Free Software Foundation, Inc.

+

+# This program is free software: you can redistribute it and/or modify

+# it under the terms of the GNU General Public License as published by

+# the Free Software Foundation, either version 3 of the License, or

+# (at your option) any later version.

+

+# This program is distributed in the hope that it will be useful,

+# but WITHOUT ANY WARRANTY; without even the implied warranty of

+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+# GNU General Public License for more details.

+

+# You should have received a copy of the GNU General Public License

+# along with this program.  If not, see <http://www.gnu.org/licenses/>.

+

+. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src

+print_ver_ dd

+

+env kill -l | grep '^INFO$' && SIGINFO='INFO' || SIGINFO='USR1'

+

+# This to avoid races in the USR1 case

+# as the dd process will terminate by default until

+# it has its handler enabled.

+trap '' $SIGINFO

+

+mkfifo_or_skip_ fifo

+

+for open in '' '1'; do

+  # Run dd with the fullblock iflag to avoid short reads

+  # which can be triggered by reception of signals

+  dd iflag=fullblock if=/dev/zero of=fifo count=100 bs=5000000 2>err & pid=$!

+

+  # Note if we sleep here we give dd a chance to exec and block on open.

+  # Otherwise we're probably testing SIG_IGN in the forked shell or early dd.

+  test "$open" && sleep .1

+

+  # dd will block on open until fifo is opened for reading.

+  # Timeout in case dd goes away erroneously which we check for below.

+  timeout 10 sh -c 'wc -c < fifo > nwritten' &

+

+  # Send lots of signals immediately to ensure dd not killed due

+  # to race setting handler, or blocking on open of fifo.

+  # Many signals also check that short reads are handled.

+  until ! kill -s $SIGINFO $pid 2>/dev/null; do

+    sleep .01

+  done

+

+  wait

+

+  # Ensure all data processed and at least last status written

+  grep '500000000 bytes .* copied' err || { cat err; fail=1; }

+done

+

+progress_output()

+{

+  { sleep "$1"; echo 1; } | dd bs=1 status=progress of=/dev/null 2>err

+  # Progress output should be for "byte ... copied", while final is "bytes ..."

+  grep 'byte .* copied' err

+}

+retry_delay_ progress_output 1 4 || { cat err; fail=1; }

+

+Exit $fail

--

cgit v0.9.0.2