title: how to check program is writing to terminal
body: <p>This is follow up to my question posted on <a href="http://codereview.stackexchange.com">codereview</a> - <a href="https://codereview.stackexchange.com/questions/120777/colorful-output-on-terminal">Colorful output on terminal</a> where I was trying to output coloured strings on terminal and detect it via <code>isatty()</code> call. However as @Jerry Coffin pointed out - </p>
  
  <blockquote>
    <p>You use isatty to check whether standard output is connected to a terminal, regardless of what stream you're writing to. This means the rest of the functions only work correctly if you pass <code>std::cout</code> as the stream to which they're going to write. Otherwise, you may allow formatting when writing to something that's not a TTY, and you may prohibit formatting when writing to something that is a TTY.</p>
  </blockquote>
  
  <p>This was something that I wasn't aware of (read as <em>had no experience in</em>) and I wasn't even aware of the fact that cin/cout can be redirected elsewhere. So I tried to read more about it and found some existing questions on SO too. Here's what I've hacked together :</p>
  
  <pre><code>// initialize them at start of program - mandatory
  
  std::streambuf const *coutbuf = std::cout.rdbuf();
  std::streambuf const *cerrbuf = std::cerr.rdbuf();
  std::streambuf const *clogbuf = std::clog.rdbuf();
  
  
  // ignore this, just checks for TERM env var
  
  inline bool supportsColor()
      {
          if(const char *env_p = std::getenv("TERM")) {
              const char *const term[8] = {
                  "xterm", "xterm-256", "xterm-256color", "vt100",
                  "color", "ansi",      "cygwin",         "linux"};
              for(unsigned int i = 0; i &lt; 8; ++i) {
                  if(std::strcmp(env_p, term[i]) == 0) return true;
              }
          }
          return false;
      }
  
  rightTerm = supportsColor();
  
  // would make necessary checks to ensure in terminal
  
  inline bool isTerminal(const std::streambuf *osbuf)
      {
          FILE *currentStream = nullptr;
          if(osbuf == coutbuf) {
              currentStream = stdout;
          }
          else if(osbuf == cerrbuf || osbuf == clogbuf) {
              currentStream = stderr;
          }
          else {
              return false;
          }
          return isatty(fileno(currentStream));
      }
  
  // this would print checking rightTerm &amp;&amp; isTerminal calls
  
  inline std::ostream &amp;operator&lt;&lt;(std::ostream &amp;os, rang::style v)
      {
          std::streambuf const *osbuf = os.rdbuf();
  
          return rightTerm &amp;&amp; isTerminal(osbuf)
                     ? os &lt;&lt; "\e[" &lt;&lt; static_cast&lt;int&gt;(v) &lt;&lt; "m"
                     : os;
      }
  </code></pre>
  
  <hr>
  
  <p><strong>My main issue is, although I've tested this manually, I'm not aware of the cases this might fail or bugs it might contain. Is this the right way to do this thing? Is there anything I might be missing?</strong></p>
  
  <hr>
  
  <p>Here's a minimal example to get running (you'll also need a <code>in.txt</code> with random data):</p>
  
  <pre><code>#include &lt;iostream&gt;
  #include &lt;fstream&gt;
  #include &lt;string&gt;
  #include &lt;unistd.h&gt;
  #include &lt;cstdlib&gt;
  #include &lt;cstring&gt;
  
  void f();
  bool supportsColor();
  
  // sample enum for foreground colors
  enum class fg : unsigned char {
      def     = 39,
      black   = 30,
      red     = 31,
      green   = 32,
      yellow  = 33,
      blue    = 34,
      magenta = 35,
      cyan    = 36,
      gray    = 37
  };
  
  // initialize them at start of program - mandatory
  // so that even if user redirects, we've a copy
  std::streambuf const *coutbuf = std::cout.rdbuf();
  std::streambuf const *cerrbuf = std::cerr.rdbuf();
  std::streambuf const *clogbuf = std::clog.rdbuf();
  
  // check if TERM supports color
  bool rightTerm = supportsColor();
  
  // Here is the implementation of isTerminal
  // which checks if program is writing to Terminal or not
  bool isTerminal(const std::streambuf *osbuf)
  {
      FILE *currentStream = nullptr;
      if(osbuf == coutbuf) {
          currentStream = stdout;
      }
      else if(osbuf == cerrbuf || osbuf == clogbuf) {
          currentStream = stderr;
      }
      else {
          return false;
      }
      return isatty(fileno(currentStream));
  }
  
  // will check if TERM supports color and isTerminal()
  inline std::ostream &amp;operator&lt;&lt;(std::ostream &amp;os, fg v)
  {
      std::streambuf const *osbuf = os.rdbuf();
  
      return rightTerm &amp;&amp; isTerminal(osbuf)
                 ? os &lt;&lt; "\e[" &lt;&lt; static_cast&lt;int&gt;(v) &lt;&lt; "m"
                 : os;
  }
  
  
  int main()
  {
  
      std::cout &lt;&lt; fg::red &lt;&lt; "ERROR HERE! " &lt;&lt; std::endl
                &lt;&lt; fg::blue &lt;&lt; "ERROR INVERSE?" &lt;&lt; std::endl;
  
      std::ifstream in("in.txt");
      std::streambuf *Orig_cinbuf = std::cin.rdbuf(); // save old buf
      std::cin.rdbuf(in.rdbuf()); // redirect std::cin to in.txt!
  
      std::ofstream out("out.txt");
      std::streambuf *Orig_coutbuf = std::cout.rdbuf(); // save old buf
      std::cout.rdbuf(out.rdbuf()); // redirect std::cout to out.txt!
  
      std::string word;
      std::cin &gt;&gt; word;                      // input from the file in.txt
      std::cout &lt;&lt; fg::blue &lt;&lt; word &lt;&lt; "  "; // output to the file out.txt
  
      f(); // call function
  
      std::cin.rdbuf(Orig_cinbuf);   // reset to standard input again
      std::cout.rdbuf(Orig_coutbuf); // reset to standard output again
  
      std::cin &gt;&gt; word;  // input from the standard input
      std::cout &lt;&lt; word; // output to the standard input
      return 0;
  }
  
  void f()
  {
      std::string line;
      while(std::getline(std::cin, line)) // input from the file in.txt
      {
          std::cout &lt;&lt; fg::green &lt;&lt; line &lt;&lt; "\n"; // output to the file out.txt
      }
  }
  
  bool supportsColor()
  {
      if(const char *env_p = std::getenv("TERM")) {
          const char *const term[8] = {"xterm",  "xterm-256", "xterm-256color",
                                       "vt100",  "color",     "ansi",
                                       "cygwin", "linux"};
          for(unsigned int i = 0; i &lt; 8; ++i) {
              if(std::strcmp(env_p, term[i]) == 0) return true;
          }
      }
      return false;
  }
  </code></pre>
  
  <p><sup>I've also tagged <code>c</code> language although this is <code>c++</code> code because the relevant code is shared b/w two and I don't want to miss any suggestions</sup></p>