Mdthbs

As a comment in I'm confused as to why "| true" in a makefile has the same effect as "|| true" user cjm wrote:

Another reason to avoid | true is that if the command produced enough output to fill up the pipe buffer, it would block waiting for true to read it.

Do we have some way of finding out what the size of the pipe buffer is?

The capacity of a pipe buffer varies across systems (and can even vary on the same system). I am not sure there is a quick, easy, and cross platform way to just lookup the capacity of a pipe.

Mac OS X, for example, uses a capacity of 16384 bytes by default, but can switch to 65336 byte capacities if large write are made to the pipe, or will switch to a capacity of a single system page if too much kernel memory is already being used by pipe buffers (see xnu/bsd/sys/pipe.h, and xnu/bsd/kern/sys_pipe.c; since these are from FreeBSD, the same behavior may happen there, too).

One Linux pipe(7) man page says that pipe capacity is 65536 bytes since Linux 2.6.11 and a single system page prior to that (e.g. 4096 bytes on (32-bit) x86 systems). The code (include/linux/pipe_fs_i.h, and fs/pipe.c) seems to use 16 system pages (i.e. 64 KiB if a system page is 4 KiB), but the buffer for each pipe can be adjusted via a fcntl on the pipe (up to a maximum capacity which defaults to 1048576 bytes, but can be changed via /proc/sys/fs/pipe-max-size)).

Here is a little bash/perl combination that I used to test the pipe capacity on my system:

#!/bin/bash

test $# -ge 1 || { echo "usage: $0 write-size [wait-time]"; exit 1; }

test $# -ge 2 || set -- "$@" 1

bytes_written=$(

{

    exec 3>&1

    {

        perl -e '

            $size = $ARGV[0];

            $block = q(a) x $size;

            $num_written = 0;

            sub report { print STDERR $num_written * $size, qq(n); }

            report; while (defined syswrite STDOUT, $block) {

                $num_written++; report;

            }

        ' "$1" 2>&3

    } | (sleep "$2"; exec 0<&-);

} | tail -1

)

printf "write size: %10d; bytes successfully before error: %dn" 

    "$1" "$bytes_written"

Here is what I found running it with various write sizes on a Mac OS X 10.6.7 system (note the change for writes larger than 16KiB):

% /bin/bash -c 'for p in {0..18}; do /tmp/ts.sh $((2 ** $p)) 0.5; done'

write size:          1; bytes successfully before error: 16384

write size:          2; bytes successfully before error: 16384

write size:          4; bytes successfully before error: 16384

write size:          8; bytes successfully before error: 16384

write size:         16; bytes successfully before error: 16384

write size:         32; bytes successfully before error: 16384

write size:         64; bytes successfully before error: 16384

write size:        128; bytes successfully before error: 16384

write size:        256; bytes successfully before error: 16384

write size:        512; bytes successfully before error: 16384

write size:       1024; bytes successfully before error: 16384

write size:       2048; bytes successfully before error: 16384

write size:       4096; bytes successfully before error: 16384

write size:       8192; bytes successfully before error: 16384

write size:      16384; bytes successfully before error: 16384

write size:      32768; bytes successfully before error: 65536

write size:      65536; bytes successfully before error: 65536

write size:     131072; bytes successfully before error: 0

write size:     262144; bytes successfully before error: 0

The same script on Linux 3.19:

/bin/bash -c 'for p in {0..18}; do /tmp/ts.sh $((2 ** $p)) 0.5; done'

write size:          1; bytes successfully before error: 65536

write size:          2; bytes successfully before error: 65536

write size:          4; bytes successfully before error: 65536

write size:          8; bytes successfully before error: 65536

write size:         16; bytes successfully before error: 65536

write size:         32; bytes successfully before error: 65536

write size:         64; bytes successfully before error: 65536

write size:        128; bytes successfully before error: 65536

write size:        256; bytes successfully before error: 65536

write size:        512; bytes successfully before error: 65536

write size:       1024; bytes successfully before error: 65536

write size:       2048; bytes successfully before error: 65536

write size:       4096; bytes successfully before error: 65536

write size:       8192; bytes successfully before error: 65536

write size:      16384; bytes successfully before error: 65536

write size:      32768; bytes successfully before error: 65536

write size:      65536; bytes successfully before error: 65536

write size:     131072; bytes successfully before error: 0

write size:     262144; bytes successfully before error: 0

Note: The PIPE_BUF value defined in the C header files (and the pathconf value for _PC_PIPE_BUF), does not specify the capacity of pipes, but the maximum number of bytes that can be written atomically (see POSIX write(2)).

Quote from include/linux/pipe_fs_i.h:

/* Differs from PIPE_BUF in that PIPE_SIZE is the length of the actual

   memory allocation, whereas PIPE_BUF makes atomicity guarantees.  */

this shell-line can show pipe buffer size too:

M=0; while true; do dd if=/dev/zero bs=1k count=1 2>/dev/null; 

       M=$(($M+1)); echo -en "r$M KB" 1>&2; done | sleep 999

(sending 1k chunks to blocked pipe until buffer full) ...some test outputs:

64K (intel-debian), 32K (aix-ppc), 64K (jslinux bellard.org)      ...Ctrl+C.

shortest bash-one-liner using printf:

M=0; while printf A; do >&2 printf "r$((++M)) B"; done | sleep 999

Here are some further alternatives to explore the actual pipe buffer capacity using shell commands only:

# get pipe buffer size using Bash

yes produce_this_string_as_output | tee >(sleep 1) | wc -c



# portable version

( (sleep 1; exec yes produce_this_string_as_output) & echo $! ) | 

     (pid=$(head -1); sleep 2; kill "$pid"; wc -c </dev/stdin)



# get buffer size of named pipe

sh -c '

  rm -f fifo

  mkfifo fifo

  yes produce_this_string_as_output | tee fifo | wc -c &

  exec 3<&- 3<fifo

  sleep 1

  exec 3<&-

  rm -f fifo

'



# Mac OS X

#getconf PIPE_BUF /

#open -e /usr/include/limits.h /usr/include/sys/pipe.h

# PIPE_SIZE

# BIG_PIPE_SIZE

# SMALL_PIPE_SIZE

# PIPE_MINDIRECT

This is a quick and dirty hack on Ubuntu 12.04, YMMV

cat >pipesize.c



#include <unistd.h>

#include <errno.h>

#include </usr/include/linux/fcntl.h>

#include <stdio.h>



void main( int argc, char *argv[] ){

  int fd ;

  long pipesize ;



  if( argc>1 ){

  // if command line arg, associate a file descriptor with it

    fprintf( stderr, "sizing %s ... ", argv[1] );

    fd = open( argv[1], O_RDONLY|O_NONBLOCK );

  }else{

  // else use STDIN as the file descriptor

    fprintf( stderr, "sizing STDIN ... " );

    fd = 0 ;

  }



  fprintf( stderr, "%ld bytesn", (long)fcntl( fd, F_GETPIPE_SZ ));

  if( errno )fprintf( stderr, "Uh oh, errno is %dn", errno );

  if( fd )close( fd );

}



gcc -o pipesize pipesize.c



mkfifo /tmp/foo



./pipesize /tmp/foo



>sizing /tmp/foo ... 65536 bytes



date | ./pipesize



>sizing STDIN ... 65536 bytes