'''
Created on 9 Mar 2015

@author: Tan Ming Sheng
'''
import json, collections, Queue, time, re
from scrapy.spider import Spider
from scrapy import signals, Request, Selector
from scrapy.contrib.loader import ItemLoader
from scrapy.item import Item, Field
from scrapy.exceptions import CloseSpider
from bs4 import BeautifulSoup
from urlparse import urljoin
from HTMLParser import HTMLParser
from pymongo import MongoClient

client = MongoClient('localhost', 27017)
LINK_NUMBER = 50

class FocraSpider(Spider):
	name = 'focras'
	'''
	To access scrapy's core API. basically can modify anything in the 'crawler'
	'''
	@classmethod
	def from_crawler(cls, crawler, **kwargs):
		print "focras - from crawler"
		spider = cls(stats=crawler.stats, settings=crawler.settings, **kwargs)
		crawler.signals.connect(spider.stopped, signals.engine_stopped)
		crawler.signals.connect(spider.idle, signals.spider_idle)
		return spider
	
	def __init__(self, stats=None, settings=None, **kwargs):
		super(FocraSpider, self).__init__(**kwargs)
		try:
			self.start_time = time.time()
			print 'focras init(' + self.cname + ') kwargs seeds ' + kwargs.get('seeds')
			print 'focras init(' + self.cname + ') kwargs template '+ self.template