#!/usr/bin/env python
#
# HTTP support for ooni-probe
# by Jacob Appelbaum <jacob@appelbaum.net>
#    Arturo Filasto' <art@fuffa.org>
#

from socket import gethostbyname
import ooni.common
import ooni.helpers
import ooni.report
import urllib2
import httplib
from urlparse import urlparse
from pprint import pprint
import pycurl
import random
import string
import re
from pprint import pprint
try:
   from BeautifulSoup import BeautifulSoup
except:
   pass                        # Never mind, let's break later.

# By default, we'll be Torbutton's UA
default_ua = { 'User-Agent' :
               'Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0' }

# Use pycurl to connect over a proxy
PROXYTYPE_SOCKS5 = 5
default_proxy_type = PROXYTYPE_SOCKS5
default_proxy_host = "127.0.0.1"
default_proxy_port = "9050"

#class HTTPResponse(object):
#  def __init__(self):


"""A very basic HTTP fetcher that uses Tor by default and returns a curl
   object."""
def http_proxy_fetch(url, headers, proxy_type=5,
                     proxy_host="127.0.0.1",
                     proxy_port=9050):
   request = pycurl.Curl()
   request.setopt(pycurl.PROXY, proxy_host)
   request.setopt(pycurl.PROXYPORT, proxy_port)
   request.setopt(pycurl.PROXYTYPE, proxy_type)
   request.setopt(pycurl.HTTPHEADER, ["User-Agent: Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.0"])
   request.setopt(pycurl.URL, url)
   response = request.perform()
   http_code = getinfo(pycurl.HTTP_CODE)
   return response, http_code

"""A very basic HTTP fetcher that returns a urllib2 response object."""
def http_fetch(url,
               headers= default_ua,
               label="generic HTTP fetch"):
   request = urllib2.Request(url, None, headers)
   response = urllib2.urlopen(request)
   return response

"""Connect to test_hostname on port 80, request url and compare it with the expected
   control_result. Optionally, a label may be set to customize
   output. If the experiment matches the control, this returns True with the http
   status code; otherwise it returns False.
"""
def http_content_match(experimental_url, control_result,
                       headers= { 'User-Agent' : default_ua },
                       label="generic HTTP content comparison"):
  request = urllib2.Request(experimental_url, None, headers)
  response = urllib2.urlopen(request)
  responseContents = response.read()
  responseCode = response.code
  if responseContents != False:
    if str(responseContents) != str(control_result):
      print label + " control " + str(control_result) + " data does not " \
            "match experiment response: " + str(responseContents)
      return False, responseCode
    return True, responseCode
  else:
    print "HTTP connection appears to have failed"
  return False, False

"""Connect to test_hostname on port 80, request url and compare it with the expected
   control_result as a regex. Optionally, a label may be set to customize
   output. If the experiment matches the control, this returns True with the HTTP
   status code; otherwise it returns False.
"""
def http_content_fuzzy_match(experimental_url, control_result,
                       headers= { 'User-Agent' : default_ua },
                       label="generic HTTP content comparison"):
  request = urllib2.Request(experimental_url, None, headers)
  response = urllib2.urlopen(request)
  responseContents = response.read()
  responseCode = response.code
  pattern = re.compile(control_result)
  match = pattern.search(responseContents)
  if responseContents != False:
    if not match:
      print label + " control " + str(control_result) + " data does not " \
            "match experiment response: " + str(responseContents)
      return False, responseCode
    return True, responseCode