'''
async fetching of urls.

Assumes robots checks have already been done.

Success returns response object and response bytes (which were already
read in order to shake out all potential network-related exceptions.)

Failure returns enough details for the caller to do something smart:
503, other 5xx, DNS fail, connect timeout, error between connect and
full response, proxy failure. Plus an errorstring good enough for logging.

'''

import time
import traceback
from collections import namedtuple
import ssl
import urllib

import asyncio
import logging
import aiohttp

from . import stats
from . import config
from . import content
from .urls import URL

LOGGER = logging.getLogger(__name__)

# these errors get printed deep in aiohttp but they also bubble up
aiohttp_errors = {
    'SSL handshake failed',
    'SSL error errno:1 reason: CERTIFICATE_VERIFY_FAILED',
    'SSL handshake failed on verifying the certificate',
    'Fatal error on transport TCPTransport',
    'Fatal error on SSL transport',
    'SSL error errno:1 reason: UNKNOWN_PROTOCOL',
    'Future exception was never retrieved',
    'Unclosed connection',
    'SSL error errno:1 reason: TLSV1_UNRECOGNIZED_NAME',