Question: I'm having difficulties dealing with character encoding. I'm trying to scrape the following url:

```
http://www.google.com/movies?near=Montreal&date=0

```

My code looks like this:

```
var http = require('http');
var url = require('url');
var Iconv  = require('iconv').Iconv;

var location = 'montreal';

var googleMovies = url.parse("http://www.google.com/movies?near=" + location);

var req = http.request(googleMovies, function(response) {
    var str = '';
    response.on('data', function(chunk) {
        str += chunk;
    });
    response.on('end', function() {

        var iconv = new Iconv('latin1', 'UTF-8');
        str = iconv.convert(str).toString();

        console.log(str);
    });
});
req.end()

```

I've first tried without:

```
    var iconv = new Iconv('latin1', 'UTF-8');
    str = iconv.convert(str).toString();

```

but that was causing the � characters. 

I've tested the source listed above on this page:

<http://nlp.fi.muni.cz/projects/chared/>

and it seem to detect it as latin1, but things could be wrong.

Answer: 
If you set your `User-Agent` to that of a desktop browser, the meta tag in the HTML and the `Content-Type` in the response headers will have the `charset` set to UTF-8 instead of latin1. Example:

```
var dest = url.parse('http://www.google.com/movies?near=montreal');
dest.headers = {
  'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36',
};

http.get(dest, function(response) {
  var str = '';

  response.on('data', function(chunk) {
    str += chunk;
  }).on('end', function() {
    console.log(str);
  }).setEncoding('utf8');
});

```
A `Buffer`'s default encoding is UTF-8, which is a variable-width encoding system. Characters after the ASCII range are encoded using multiple bytes. If you're receiving latin1-specific characters (codepoints > 127), they'll have the first bit set, which a UTF-8 decoder will consider as a multi-byte character, eventually resulting in an unmapped codepoint (displayed as �). 

`iconv` has a streaming decoder that you can pipe your response stream to.

```
http.request(googleMovies, function (response) {
    var iconv = new Iconv('latin1', 'UTF-8');
    response.pipe(iconv).pipe(process.stdout);
    //or response.pipe(iconv).on('data', console.log);
}).end();

```