def get_content(url, headers={}, decoded=True):
    """Gets the content of a URL via sending a HTTP GET request.

    Args:
        url: A URL.
        headers: Request headers used by the client.
        decoded: Whether decode the response body using UTF-8 or the charset specified in Content-Type.

    Returns:
        The content as a string.
    """

    logging.debug('get_content: %s' % url)

    req = request.Request(url, headers=headers)
    if cookies:
        cookies.add_cookie_header(req)
        req.headers.update(req.unredirected_hdrs)

    response = urlopen_with_retry(req)
    data = response.read()

    # Handle HTTP compression for gzip and deflate (zlib)
    content_encoding = response.getheader('Content-Encoding')
    if content_encoding == 'gzip':
        data = ungzip(data)
    elif content_encoding == 'deflate':
        data = undeflate(data)

    # Decode the response body
    if decoded:
        charset = match1(
            response.getheader('Content-Type', ''), r'charset=([\w-]+)'
        )
        if charset is not None:
            data = data.decode(charset, 'ignore')
        else:
            data = data.decode('utf-8', 'ignore')

    return data
def get_content(url, headers={}, decoded=True):
    

    logging.debug('get_content: %s' % url)

    req = request.Request(url, headers=headers)
    if cookies:
        cookies.add_cookie_header(req)
        req.headers.update(req.unredirected_hdrs)

    response = urlopen_with_retry(req)
    data = response.read()

    
    content_encoding = response.getheader('Content-Encoding')
    if content_encoding == 'gzip':
        data = ungzip(data)
    elif content_encoding == 'deflate':
        data = undeflate(data)

    
    if decoded:
        charset = match1(
            response.getheader('Content-Type', ''), r'charset=([\w-]+)'
        )
        if charset is not None:
            data = data.decode(charset, 'ignore')
        else:
            data = data.decode('utf-8', 'ignore')

    return data
def get_content ( url , headers = { } , decoded = True ) : logging . debug ( 'get_content: %s' % url ) req = request . Request ( url , headers = headers ) if cookies : cookies . add_cookie_header ( req ) req . headers . update ( req . unredirected_hdrs ) response = urlopen_with_retry ( req ) data = response . read ( ) # Handle HTTP compression for gzip and deflate (zlib) content_encoding = response . getheader ( 'Content-Encoding' ) if content_encoding == 'gzip' : data = ungzip ( data ) elif content_encoding == 'deflate' : data = undeflate ( data ) # Decode the response body if decoded : charset = match1 ( response . getheader ( 'Content-Type' , '' ) , r'charset=([\w-]+)' ) if charset is not None : data = data . decode ( charset , 'ignore' ) else : data = data . decode ( 'utf-8' , 'ignore' ) return data
Gets the content of a URL via sending a HTTP GET request.

    Args:
        url: A URL.
        headers: Request headers used by the client.
        decoded: Whether decode the response body using UTF-8 or the charset specified in Content-Type.

    Returns:
        The content as a string.
Gets the content of a URL via sending a HTTP GET request .